apunkt/ktx
1
0
Fork 0
mirror of https://github.com/Kaelio/ktx.git synced 2026-06-22 08:38:08 +02:00
Code Issues Projects Releases Packages Wiki Activity

Merge remote-tracking branch 'origin/main' into scan-during-setup

Browse source 
# Conflicts:
#	packages/cli/src/setup-context.test.ts
#	packages/cli/src/setup-context.ts
#	packages/cli/src/setup.test.ts
#	packages/cli/src/setup.ts
This commit is contained in:
Luca Martial 2026-05-13 09:25:25 -07:00
commit fe0a59f55e
357 changed files with 14537 additions and 14297 deletions
Download patch file Download diff file Expand all files Collapse all files

32
packages/cli/assets/demo/orbit/links/provenance.json
View file

@ -2,7 +2,7 @@
{
"id": "link-001",
"artifactKind": "wiki",
"artifactKey": "knowledge/global/arr-contract-first.md",
"artifactKey": "wiki/global/arr-contract-first.md",
"sourceKind": "warehouse",
"sourcePath": "contracts",
"relationship": "describes",
@ -11,7 +11,7 @@
{
"id": "link-002",
"artifactKind": "wiki",
"artifactKey": "knowledge/global/arr-contract-first.md",
"artifactKey": "wiki/global/arr-contract-first.md",
"sourceKind": "notion",
"sourcePath": "raw-sources/notion/arr-and-contract-reporting-notes.md",
"relationship": "derived_from",
@ -20,7 +20,7 @@
{
"id": "link-003",
"artifactKind": "wiki",
"artifactKey": "knowledge/global/revenue-gross-to-net.md",
"artifactKey": "wiki/global/revenue-gross-to-net.md",
"sourceKind": "warehouse",
"sourcePath": "invoices",
"relationship": "describes",
@ -29,7 +29,7 @@
{
"id": "link-004",
"artifactKind": "wiki",
"artifactKey": "knowledge/global/revenue-gross-to-net.md",
"artifactKey": "wiki/global/revenue-gross-to-net.md",
"sourceKind": "notion",
"sourcePath": "raw-sources/notion/revenue-reporting-policy.md",
"relationship": "derived_from",
@ -38,7 +38,7 @@
{
"id": "link-005",
"artifactKind": "wiki",
"artifactKey": "knowledge/global/discount-expiration.md",
"artifactKey": "wiki/global/discount-expiration.md",
"sourceKind": "warehouse",
"sourcePath": "arr_movements",
"relationship": "describes",
@ -47,7 +47,7 @@
{
"id": "link-006",
"artifactKind": "wiki",
"artifactKey": "knowledge/global/nrr-retention.md",
"artifactKey": "wiki/global/nrr-retention.md",
"sourceKind": "warehouse",
"sourcePath": "arr_movements",
"relationship": "describes",
@ -56,7 +56,7 @@
{
"id": "link-007",
"artifactKind": "wiki",
"artifactKey": "knowledge/global/nrr-retention.md",
"artifactKey": "wiki/global/nrr-retention.md",
"sourceKind": "notion",
"sourcePath": "raw-sources/notion/retention-and-nrr-definition-notes.md",
"relationship": "derived_from",
@ -65,7 +65,7 @@
{
"id": "link-008",
"artifactKind": "wiki",
"artifactKey": "knowledge/global/nrr-retention.md",
"artifactKey": "wiki/global/nrr-retention.md",
"sourceKind": "bi",
"sourcePath": "raw-sources/bi/account_retention.view.lkml",
"relationship": "derived_from",
@ -74,7 +74,7 @@
{
"id": "link-009",
"artifactKind": "wiki",
"artifactKey": "knowledge/global/segment-classification.md",
"artifactKey": "wiki/global/segment-classification.md",
"sourceKind": "warehouse",
"sourcePath": "plans",
"relationship": "describes",
@ -83,7 +83,7 @@
{
"id": "link-010",
"artifactKind": "wiki",
"artifactKey": "knowledge/global/segment-classification.md",
"artifactKey": "wiki/global/segment-classification.md",
"sourceKind": "notion",
"sourcePath": "raw-sources/notion/sales-ops-segmentation-guide.md",
"relationship": "derived_from",
@ -92,7 +92,7 @@
{
"id": "link-011",
"artifactKind": "wiki",
"artifactKey": "knowledge/global/activation-policy.md",
"artifactKey": "wiki/global/activation-policy.md",
"sourceKind": "notion",
"sourcePath": "raw-sources/notion/activation-policy-decision-record.md",
"relationship": "derived_from",
@ -101,7 +101,7 @@
{
"id": "link-012",
"artifactKind": "wiki",
"artifactKey": "knowledge/global/procurement-workflows.md",
"artifactKey": "wiki/global/procurement-workflows.md",
"sourceKind": "warehouse",
"sourcePath": "purchase_requests",
"relationship": "describes",
@ -110,7 +110,7 @@
{
"id": "link-013",
"artifactKind": "wiki",
"artifactKey": "knowledge/global/customer-health-scoring.md",
"artifactKey": "wiki/global/customer-health-scoring.md",
"sourceKind": "notion",
"sourcePath": "raw-sources/notion/customer-health-playbook.md",
"relationship": "derived_from",
@ -119,7 +119,7 @@
{
"id": "link-014",
"artifactKind": "wiki",
"artifactKey": "knowledge/global/customer-health-scoring.md",
"artifactKey": "wiki/global/customer-health-scoring.md",
"sourceKind": "warehouse",
"sourcePath": "support_tickets",
"relationship": "describes",
@ -128,7 +128,7 @@
{
"id": "link-015",
"artifactKind": "wiki",
"artifactKey": "knowledge/global/support-escalation.md",
"artifactKey": "wiki/global/support-escalation.md",
"sourceKind": "notion",
"sourcePath": "raw-sources/notion/support-escalation-runbook.md",
"relationship": "derived_from",
@ -137,7 +137,7 @@
{
"id": "link-016",
"artifactKind": "wiki",
"artifactKey": "knowledge/global/internal-test-exclusion.md",
"artifactKey": "wiki/global/internal-test-exclusion.md",
"sourceKind": "notion",
"sourcePath": "raw-sources/notion/analyst-onboarding.md",
"relationship": "derived_from",

2
packages/cli/assets/demo/orbit/manifest.json
View file

@ -47,7 +47,7 @@
"sourceCount": 46
},
"knowledge": {
"path": "knowledge/global",
"path": "wiki/global",
"pageCount": 28
},
"links": {

82
packages/cli/assets/demo/orbit/replay.memory-flow.v1.json
View file

@ -71,7 +71,7 @@
"type": "work_unit_started",
"unitKey": "revenue-and-contracts",
"skills": [
"knowledge_capture",
"wiki_capture",
"sl_capture"
],
"stepBudget": 40
@ -81,21 +81,21 @@
"unitKey": "revenue-and-contracts",
"target": "wiki",
"action": "created",
"key": "knowledge/global/arr-contract-first.md"
"key": "wiki/global/arr-contract-first.md"
},
{
"type": "candidate_action",
"unitKey": "revenue-and-contracts",
"target": "wiki",
"action": "created",
"key": "knowledge/global/revenue-gross-to-net.md"
"key": "wiki/global/revenue-gross-to-net.md"
},
{
"type": "candidate_action",
"unitKey": "revenue-and-contracts",
"target": "wiki",
"action": "created",
"key": "knowledge/global/discount-expiration.md"
"key": "wiki/global/discount-expiration.md"
},
{
"type": "candidate_action",
@ -127,7 +127,7 @@
"type": "work_unit_started",
"unitKey": "retention-and-segments",
"skills": [
"knowledge_capture",
"wiki_capture",
"sl_capture"
],
"stepBudget": 40
@ -137,14 +137,14 @@
"unitKey": "retention-and-segments",
"target": "wiki",
"action": "created",
"key": "knowledge/global/nrr-retention.md"
"key": "wiki/global/nrr-retention.md"
},
{
"type": "candidate_action",
"unitKey": "retention-and-segments",
"target": "wiki",
"action": "created",
"key": "knowledge/global/segment-classification.md"
"key": "wiki/global/segment-classification.md"
},
{
"type": "candidate_action",
@ -162,7 +162,7 @@
"type": "work_unit_started",
"unitKey": "procurement-and-activation",
"skills": [
"knowledge_capture",
"wiki_capture",
"sl_capture"
],
"stepBudget": 40
@ -172,14 +172,14 @@
"unitKey": "procurement-and-activation",
"target": "wiki",
"action": "created",
"key": "knowledge/global/activation-policy.md"
"key": "wiki/global/activation-policy.md"
},
{
"type": "candidate_action",
"unitKey": "procurement-and-activation",
"target": "wiki",
"action": "created",
"key": "knowledge/global/procurement-workflows.md"
"key": "wiki/global/procurement-workflows.md"
},
{
"type": "candidate_action",
@ -197,7 +197,7 @@
"type": "work_unit_started",
"unitKey": "support-and-health",
"skills": [
"knowledge_capture",
"wiki_capture",
"sl_capture"
],
"stepBudget": 40
@ -207,14 +207,14 @@
"unitKey": "support-and-health",
"target": "wiki",
"action": "created",
"key": "knowledge/global/customer-health-scoring.md"
"key": "wiki/global/customer-health-scoring.md"
},
{
"type": "candidate_action",
"unitKey": "support-and-health",
"target": "wiki",
"action": "created",
"key": "knowledge/global/support-escalation.md"
"key": "wiki/global/support-escalation.md"
},
{
"type": "candidate_action",
@ -232,7 +232,7 @@
"type": "work_unit_started",
"unitKey": "governance-and-exclusions",
"skills": [
"knowledge_capture"
"wiki_capture"
],
"stepBudget": 40
},
@ -241,7 +241,7 @@
"unitKey": "governance-and-exclusions",
"target": "wiki",
"action": "created",
"key": "knowledge/global/internal-test-exclusion.md"
"key": "wiki/global/internal-test-exclusion.md"
},
{
"type": "work_unit_finished",
@ -321,7 +321,7 @@
"unitKey": "revenue-and-contracts",
"target": "wiki",
"action": "created",
"key": "knowledge/global/arr-contract-first.md",
"key": "wiki/global/arr-contract-first.md",
"summary": "ARR follows contract precedence with cancellation and discount caveats.",
"rawFiles": [
"contracts",
@ -334,7 +334,7 @@
"unitKey": "revenue-and-contracts",
"target": "wiki",
"action": "created",
"key": "knowledge/global/revenue-gross-to-net.md",
"key": "wiki/global/revenue-gross-to-net.md",
"summary": "Invoice, refund, and revenue dashboard evidence reconcile gross to net revenue.",
"rawFiles": [
"invoices",
@ -346,7 +346,7 @@
"unitKey": "revenue-and-contracts",
"target": "wiki",
"action": "created",
"key": "knowledge/global/discount-expiration.md",
"key": "wiki/global/discount-expiration.md",
"summary": "Discount expiration is separated from organic contraction for retention reporting.",
"rawFiles": [
"contracts",
@ -394,7 +394,7 @@
"unitKey": "retention-and-segments",
"target": "wiki",
"action": "created",
"key": "knowledge/global/nrr-retention.md",
"key": "wiki/global/nrr-retention.md",
"summary": "NRR uses parent-account rollups and quarterly ARR movement windows.",
"rawFiles": [
"accounts",
@ -407,7 +407,7 @@
"unitKey": "retention-and-segments",
"target": "wiki",
"action": "created",
"key": "knowledge/global/segment-classification.md",
"key": "wiki/global/segment-classification.md",
"summary": "Segment labels come from plan mapping and sales-ops policy notes.",
"rawFiles": [
"accounts",
@ -432,7 +432,7 @@
"unitKey": "procurement-and-activation",
"target": "wiki",
"action": "created",
"key": "knowledge/global/activation-policy.md",
"key": "wiki/global/activation-policy.md",
"summary": "Activation policy changed on January 15, 2026 and is encoded for agents.",
"rawFiles": [
"purchase_requests",
@ -445,7 +445,7 @@
"unitKey": "procurement-and-activation",
"target": "wiki",
"action": "created",
"key": "knowledge/global/procurement-workflows.md",
"key": "wiki/global/procurement-workflows.md",
"summary": "Procurement requester activity and approval events explain product usage.",
"rawFiles": [
"purchase_requests",
@ -468,7 +468,7 @@
"unitKey": "support-and-health",
"target": "wiki",
"action": "created",
"key": "knowledge/global/customer-health-scoring.md",
"key": "wiki/global/customer-health-scoring.md",
"summary": "Customer health combines support severity, ARR exposure, and product usage.",
"rawFiles": [
"support_tickets",
@ -480,7 +480,7 @@
"unitKey": "support-and-health",
"target": "wiki",
"action": "created",
"key": "knowledge/global/support-escalation.md",
"key": "wiki/global/support-escalation.md",
"summary": "Escalation tiers map ticket severity to SLA expectations.",
"rawFiles": [
"support_tickets",
@ -503,7 +503,7 @@
"unitKey": "governance-and-exclusions",
"target": "wiki",
"action": "created",
"key": "knowledge/global/internal-test-exclusion.md",
"key": "wiki/global/internal-test-exclusion.md",
"summary": "Canonical metrics exclude internal and test accounts across source families.",
"rawFiles": [
"raw-sources/notion/analyst-onboarding.md"
@ -515,97 +515,97 @@
{
"rawPath": "contracts",
"artifactKind": "wiki",
"artifactKey": "knowledge/global/arr-contract-first.md",
"artifactKey": "wiki/global/arr-contract-first.md",
"actionType": "wiki_written"
},
{
"rawPath": "raw-sources/notion/arr-and-contract-reporting-notes.md",
"artifactKind": "wiki",
"artifactKey": "knowledge/global/arr-contract-first.md",
"artifactKey": "wiki/global/arr-contract-first.md",
"actionType": "wiki_written"
},
{
"rawPath": "invoices",
"artifactKind": "wiki",
"artifactKey": "knowledge/global/revenue-gross-to-net.md",
"artifactKey": "wiki/global/revenue-gross-to-net.md",
"actionType": "wiki_written"
},
{
"rawPath": "raw-sources/notion/revenue-reporting-policy.md",
"artifactKind": "wiki",
"artifactKey": "knowledge/global/revenue-gross-to-net.md",
"artifactKey": "wiki/global/revenue-gross-to-net.md",
"actionType": "wiki_written"
},
{
"rawPath": "arr_movements",
"artifactKind": "wiki",
"artifactKey": "knowledge/global/discount-expiration.md",
"artifactKey": "wiki/global/discount-expiration.md",
"actionType": "wiki_written"
},
{
"rawPath": "arr_movements",
"artifactKind": "wiki",
"artifactKey": "knowledge/global/nrr-retention.md",
"artifactKey": "wiki/global/nrr-retention.md",
"actionType": "wiki_written"
},
{
"rawPath": "raw-sources/notion/retention-and-nrr-definition-notes.md",
"artifactKind": "wiki",
"artifactKey": "knowledge/global/nrr-retention.md",
"artifactKey": "wiki/global/nrr-retention.md",
"actionType": "wiki_written"
},
{
"rawPath": "raw-sources/bi/account_retention.view.lkml",
"artifactKind": "wiki",
"artifactKey": "knowledge/global/nrr-retention.md",
"artifactKey": "wiki/global/nrr-retention.md",
"actionType": "wiki_written"
},
{
"rawPath": "plans",
"artifactKind": "wiki",
"artifactKey": "knowledge/global/segment-classification.md",
"artifactKey": "wiki/global/segment-classification.md",
"actionType": "wiki_written"
},
{
"rawPath": "raw-sources/notion/sales-ops-segmentation-guide.md",
"artifactKind": "wiki",
"artifactKey": "knowledge/global/segment-classification.md",
"artifactKey": "wiki/global/segment-classification.md",
"actionType": "wiki_written"
},
{
"rawPath": "raw-sources/notion/activation-policy-decision-record.md",
"artifactKind": "wiki",
"artifactKey": "knowledge/global/activation-policy.md",
"artifactKey": "wiki/global/activation-policy.md",
"actionType": "wiki_written"
},
{
"rawPath": "purchase_requests",
"artifactKind": "wiki",
"artifactKey": "knowledge/global/procurement-workflows.md",
"artifactKey": "wiki/global/procurement-workflows.md",
"actionType": "wiki_written"
},
{
"rawPath": "raw-sources/notion/customer-health-playbook.md",
"artifactKind": "wiki",
"artifactKey": "knowledge/global/customer-health-scoring.md",
"artifactKey": "wiki/global/customer-health-scoring.md",
"actionType": "wiki_written"
},
{
"rawPath": "support_tickets",
"artifactKind": "wiki",
"artifactKey": "knowledge/global/customer-health-scoring.md",
"artifactKey": "wiki/global/customer-health-scoring.md",
"actionType": "wiki_written"
},
{
"rawPath": "raw-sources/notion/support-escalation-runbook.md",
"artifactKind": "wiki",
"artifactKey": "knowledge/global/support-escalation.md",
"artifactKey": "wiki/global/support-escalation.md",
"actionType": "wiki_written"
},
{
"rawPath": "raw-sources/notion/analyst-onboarding.md",
"artifactKind": "wiki",
"artifactKey": "knowledge/global/internal-test-exclusion.md",
"artifactKey": "wiki/global/internal-test-exclusion.md",
"actionType": "wiki_written"
},
{

0
packages/cli/assets/demo/orbit/knowledge/global/.gitkeep → packages/cli/assets/demo/orbit/wiki/global/.gitkeep
View file

0
packages/cli/assets/demo/orbit/knowledge/global/customer-communication-policy.md → packages/cli/assets/demo/orbit/wiki/global/customer-communication-policy.md
View file

0
packages/cli/assets/demo/orbit/knowledge/global/new-hire-onboarding-policy.md → packages/cli/assets/demo/orbit/wiki/global/new-hire-onboarding-policy.md
View file

0
packages/cli/assets/demo/orbit/knowledge/global/orbit-activation-kpi-glossary.md → packages/cli/assets/demo/orbit/wiki/global/orbit-activation-kpi-glossary.md
View file

0
packages/cli/assets/demo/orbit/knowledge/global/orbit-activation-policy-change-jan-2026.md → packages/cli/assets/demo/orbit/wiki/global/orbit-activation-policy-change-jan-2026.md
View file

0
packages/cli/assets/demo/orbit/knowledge/global/orbit-arr-contract-first-definition.md → packages/cli/assets/demo/orbit/wiki/global/orbit-arr-contract-first-definition.md
View file

0
packages/cli/assets/demo/orbit/knowledge/global/orbit-company-overview.md → packages/cli/assets/demo/orbit/wiki/global/orbit-company-overview.md
View file

0
packages/cli/assets/demo/orbit/knowledge/global/orbit-customer-health-risk-definition.md → packages/cli/assets/demo/orbit/wiki/global/orbit-customer-health-risk-definition.md
View file

0
packages/cli/assets/demo/orbit/knowledge/global/orbit-customer-stakeholder-needs.md → packages/cli/assets/demo/orbit/wiki/global/orbit-customer-stakeholder-needs.md
View file

2
packages/cli/assets/demo/orbit/knowledge/global/orbit-customers-source.md → packages/cli/assets/demo/orbit/wiki/global/orbit-customers-source.md
View file

@ -57,4 +57,4 @@ Always join through `customer.id`. Do not join on `email`.
- **Join key:** Always use `customer.id`, never `email`.
- **Timezone:** `created_at` and `last_seen_at` are UTC. Confirm whether a question expects UTC or a local business day before filtering.
- **Paying vs. all:** `free` customers must be excluded from paying-customer follow-ups. Use `paying_customer_count`, not `customer_count`.
- **plan_tier values:** `free`, `pro`, `enterprise`. Note: `pro_plus` is a legacy alias for `growth` in the account/contract layer (see `orbit-plan-segment-normalization`), but `plan_tier` on this table uses `pro` not `pro_plus`.
- **plan_tier values:** `free`, `pro`, `enterprise`. Note: use the canonical plan names from the account/contract layer (see `orbit-plan-segment-normalization`); `plan_tier` on this table uses `pro` rather than `growth`.

0
packages/cli/assets/demo/orbit/knowledge/global/orbit-dbt-exposures.md → packages/cli/assets/demo/orbit/wiki/global/orbit-dbt-exposures.md
View file

0
packages/cli/assets/demo/orbit/knowledge/global/orbit-dbt-project-overview.md → packages/cli/assets/demo/orbit/wiki/global/orbit-dbt-project-overview.md
View file

0
packages/cli/assets/demo/orbit/knowledge/global/orbit-how-we-work.md → packages/cli/assets/demo/orbit/wiki/global/orbit-how-we-work.md
View file

0
packages/cli/assets/demo/orbit/knowledge/global/orbit-known-product-gaps.md → packages/cli/assets/demo/orbit/wiki/global/orbit-known-product-gaps.md
View file

0
packages/cli/assets/demo/orbit/knowledge/global/orbit-mart-account-activity.md → packages/cli/assets/demo/orbit/wiki/global/orbit-mart-account-activity.md
View file

0
packages/cli/assets/demo/orbit/knowledge/global/orbit-mart-account-segments.md → packages/cli/assets/demo/orbit/wiki/global/orbit-mart-account-segments.md
View file

0
packages/cli/assets/demo/orbit/knowledge/global/orbit-mart-arr-daily.md → packages/cli/assets/demo/orbit/wiki/global/orbit-mart-arr-daily.md
View file

0
packages/cli/assets/demo/orbit/knowledge/global/orbit-mart-nrr-quarterly.md → packages/cli/assets/demo/orbit/wiki/global/orbit-mart-nrr-quarterly.md
View file

0
packages/cli/assets/demo/orbit/knowledge/global/orbit-mart-procurement-activity.md → packages/cli/assets/demo/orbit/wiki/global/orbit-mart-procurement-activity.md
View file

0
packages/cli/assets/demo/orbit/knowledge/global/orbit-mart-retention-movement-breakout.md → packages/cli/assets/demo/orbit/wiki/global/orbit-mart-retention-movement-breakout.md
View file

0
packages/cli/assets/demo/orbit/knowledge/global/orbit-mart-revenue-daily.md → packages/cli/assets/demo/orbit/wiki/global/orbit-mart-revenue-daily.md
View file

0
packages/cli/assets/demo/orbit/knowledge/global/orbit-metabase-sql-library-patterns.md → packages/cli/assets/demo/orbit/wiki/global/orbit-metabase-sql-library-patterns.md
View file

0
packages/cli/assets/demo/orbit/knowledge/global/orbit-nrr-discount-expiration-treatment.md → packages/cli/assets/demo/orbit/wiki/global/orbit-nrr-discount-expiration-treatment.md
View file

0
packages/cli/assets/demo/orbit/knowledge/global/orbit-plan-segment-normalization.md → packages/cli/assets/demo/orbit/wiki/global/orbit-plan-segment-normalization.md
View file

0
packages/cli/assets/demo/orbit/knowledge/global/orbit-procurement-qualifying-actions.md → packages/cli/assets/demo/orbit/wiki/global/orbit-procurement-qualifying-actions.md
View file

0
packages/cli/assets/demo/orbit/knowledge/global/orbit-product-design-principles.md → packages/cli/assets/demo/orbit/wiki/global/orbit-product-design-principles.md
View file

0
packages/cli/assets/demo/orbit/knowledge/global/orbit-product-review-checklist.md → packages/cli/assets/demo/orbit/wiki/global/orbit-product-review-checklist.md
View file

0
packages/cli/assets/demo/orbit/knowledge/global/orbit-revenue-gross-to-net-reconciliation.md → packages/cli/assets/demo/orbit/wiki/global/orbit-revenue-gross-to-net-reconciliation.md
View file

2
packages/cli/assets/demo/orbit/knowledge/global/sales-ops-cs-handoff-process.md → packages/cli/assets/demo/orbit/wiki/global/sales-ops-cs-handoff-process.md
View file

@ -27,7 +27,7 @@ Sales Ops must complete the handoff **before the first implementation call**. Cu
| Field | Notes |
|---|---|
| Current plan | Starter / Growth / Enterprise — use canonical plan name, not legacy aliases |
| Current plan | Starter / Growth / Enterprise — use canonical plan name |
| Account segment | self_serve / commercial / enterprise (see `orbit-plan-segment-normalization`) |
| Contract shape | Term, ARR, any discounts or custom terms |
| Renewal contact | Named person on the customer side responsible for renewal |

88
packages/cli/scripts/build-demo-assets.mjs
View file

@ -229,39 +229,39 @@ const knowledgePages = [
];
const provenanceLinks = [
['wiki', 'knowledge/global/arr-contract-first.md', 'warehouse', 'contracts', 'describes', 1],
['wiki', 'wiki/global/arr-contract-first.md', 'warehouse', 'contracts', 'describes', 1],
[
'wiki',
'knowledge/global/arr-contract-first.md',
'wiki/global/arr-contract-first.md',
'notion',
'raw-sources/notion/arr-and-contract-reporting-notes.md',
'derived_from',
0.95,
],
['wiki', 'knowledge/global/revenue-gross-to-net.md', 'warehouse', 'invoices', 'describes', 1],
['wiki', 'wiki/global/revenue-gross-to-net.md', 'warehouse', 'invoices', 'describes', 1],
[
'wiki',
'knowledge/global/revenue-gross-to-net.md',
'wiki/global/revenue-gross-to-net.md',
'notion',
'raw-sources/notion/revenue-reporting-policy.md',
'derived_from',
0.95,
],
['wiki', 'knowledge/global/discount-expiration.md', 'warehouse', 'arr_movements', 'describes', 1],
['wiki', 'knowledge/global/nrr-retention.md', 'warehouse', 'arr_movements', 'describes', 1],
['wiki', 'wiki/global/discount-expiration.md', 'warehouse', 'arr_movements', 'describes', 1],
['wiki', 'wiki/global/nrr-retention.md', 'warehouse', 'arr_movements', 'describes', 1],
[
'wiki',
'knowledge/global/nrr-retention.md',
'wiki/global/nrr-retention.md',
'notion',
'raw-sources/notion/retention-and-nrr-definition-notes.md',
'derived_from',
0.95,
],
['wiki', 'knowledge/global/nrr-retention.md', 'bi', 'raw-sources/bi/account_retention.view.lkml', 'derived_from', 0.85],
['wiki', 'knowledge/global/segment-classification.md', 'warehouse', 'plans', 'describes', 1],
['wiki', 'wiki/global/nrr-retention.md', 'bi', 'raw-sources/bi/account_retention.view.lkml', 'derived_from', 0.85],
['wiki', 'wiki/global/segment-classification.md', 'warehouse', 'plans', 'describes', 1],
[
'wiki',
'knowledge/global/segment-classification.md',
'wiki/global/segment-classification.md',
'notion',
'raw-sources/notion/sales-ops-segmentation-guide.md',
'derived_from',
@ -269,25 +269,25 @@ const provenanceLinks = [
],
[
'wiki',
'knowledge/global/activation-policy.md',
'wiki/global/activation-policy.md',
'notion',
'raw-sources/notion/activation-policy-decision-record.md',
'derived_from',
0.95,
],
['wiki', 'knowledge/global/procurement-workflows.md', 'warehouse', 'purchase_requests', 'describes', 1],
['wiki', 'wiki/global/procurement-workflows.md', 'warehouse', 'purchase_requests', 'describes', 1],
[
'wiki',
'knowledge/global/customer-health-scoring.md',
'wiki/global/customer-health-scoring.md',
'notion',
'raw-sources/notion/customer-health-playbook.md',
'derived_from',
0.9,
],
['wiki', 'knowledge/global/customer-health-scoring.md', 'warehouse', 'support_tickets', 'describes', 1],
['wiki', 'wiki/global/customer-health-scoring.md', 'warehouse', 'support_tickets', 'describes', 1],
[
'wiki',
'knowledge/global/support-escalation.md',
'wiki/global/support-escalation.md',
'notion',
'raw-sources/notion/support-escalation-runbook.md',
'derived_from',
@ -295,7 +295,7 @@ const provenanceLinks = [
],
[
'wiki',
'knowledge/global/internal-test-exclusion.md',
'wiki/global/internal-test-exclusion.md',
'notion',
'raw-sources/notion/analyst-onboarding.md',
'derived_from',
@ -490,7 +490,7 @@ function buildActions() {
unitKey: 'revenue-and-contracts',
target: 'wiki',
action: 'created',
key: 'knowledge/global/arr-contract-first.md',
key: 'wiki/global/arr-contract-first.md',
summary: 'ARR follows contract precedence with cancellation and discount caveats.',
rawFiles: ['contracts', 'arr_movements', 'raw-sources/notion/arr-and-contract-reporting-notes.md'],
status: 'success',
@ -499,7 +499,7 @@ function buildActions() {
unitKey: 'revenue-and-contracts',
target: 'wiki',
action: 'created',
key: 'knowledge/global/revenue-gross-to-net.md',
key: 'wiki/global/revenue-gross-to-net.md',
summary: 'Invoice, refund, and revenue dashboard evidence reconcile gross to net revenue.',
rawFiles: ['invoices', 'raw-sources/bi/revenue_exec.dashboard.lookml'],
status: 'success',
@ -508,7 +508,7 @@ function buildActions() {
unitKey: 'revenue-and-contracts',
target: 'wiki',
action: 'created',
key: 'knowledge/global/discount-expiration.md',
key: 'wiki/global/discount-expiration.md',
summary: 'Discount expiration is separated from organic contraction for retention reporting.',
rawFiles: ['contracts', 'arr_movements'],
status: 'success',
@ -544,7 +544,7 @@ function buildActions() {
unitKey: 'retention-and-segments',
target: 'wiki',
action: 'created',
key: 'knowledge/global/nrr-retention.md',
key: 'wiki/global/nrr-retention.md',
summary: 'NRR uses parent-account rollups and quarterly ARR movement windows.',
rawFiles: ['accounts', 'arr_movements', 'raw-sources/notion/retention-and-nrr-definition-notes.md'],
status: 'success',
@ -553,7 +553,7 @@ function buildActions() {
unitKey: 'retention-and-segments',
target: 'wiki',
action: 'created',
key: 'knowledge/global/segment-classification.md',
key: 'wiki/global/segment-classification.md',
summary: 'Segment labels come from plan mapping and sales-ops policy notes.',
rawFiles: ['accounts', 'plans', 'raw-sources/notion/sales-ops-segmentation-guide.md'],
status: 'success',
@ -571,7 +571,7 @@ function buildActions() {
unitKey: 'procurement-and-activation',
target: 'wiki',
action: 'created',
key: 'knowledge/global/activation-policy.md',
key: 'wiki/global/activation-policy.md',
summary: 'Activation policy changed on January 15, 2026 and is encoded for agents.',
rawFiles: ['purchase_requests', 'users', 'raw-sources/notion/activation-policy-decision-record.md'],
status: 'success',
@ -580,7 +580,7 @@ function buildActions() {
unitKey: 'procurement-and-activation',
target: 'wiki',
action: 'created',
key: 'knowledge/global/procurement-workflows.md',
key: 'wiki/global/procurement-workflows.md',
summary: 'Procurement requester activity and approval events explain product usage.',
rawFiles: ['purchase_requests', 'raw-sources/bi/procurement_activity.view.lkml'],
status: 'success',
@ -598,7 +598,7 @@ function buildActions() {
unitKey: 'support-and-health',
target: 'wiki',
action: 'created',
key: 'knowledge/global/customer-health-scoring.md',
key: 'wiki/global/customer-health-scoring.md',
summary: 'Customer health combines support severity, ARR exposure, and product usage.',
rawFiles: ['support_tickets', 'raw-sources/notion/customer-health-playbook.md'],
status: 'success',
@ -607,7 +607,7 @@ function buildActions() {
unitKey: 'support-and-health',
target: 'wiki',
action: 'created',
key: 'knowledge/global/support-escalation.md',
key: 'wiki/global/support-escalation.md',
summary: 'Escalation tiers map ticket severity to SLA expectations.',
rawFiles: ['support_tickets', 'raw-sources/notion/support-escalation-runbook.md'],
status: 'success',
@ -625,7 +625,7 @@ function buildActions() {
unitKey: 'governance-and-exclusions',
target: 'wiki',
action: 'created',
key: 'knowledge/global/internal-test-exclusion.md',
key: 'wiki/global/internal-test-exclusion.md',
summary: 'Canonical metrics exclude internal and test accounts across source families.',
rawFiles: ['raw-sources/notion/analyst-onboarding.md'],
status: 'success',
@ -665,27 +665,27 @@ function buildReplay(provenance, transcripts) {
{ type: 'raw_snapshot_written', syncId: 'demo-seeded-sync', rawFileCount: 29 },
{ type: 'diff_computed', added: 29, modified: 0, deleted: 0, unchanged: 0 },
{ type: 'chunks_planned', chunkCount: 5, workUnitCount: 5, evictionCount: 0 },
{ type: 'work_unit_started', unitKey: 'revenue-and-contracts', skills: ['knowledge_capture', 'sl_capture'], stepBudget: 40 },
{ type: 'work_unit_started', unitKey: 'revenue-and-contracts', skills: ['wiki_capture', 'sl_capture'], stepBudget: 40 },
{
type: 'candidate_action',
unitKey: 'revenue-and-contracts',
target: 'wiki',
action: 'created',
key: 'knowledge/global/arr-contract-first.md',
key: 'wiki/global/arr-contract-first.md',
},
{
type: 'candidate_action',
unitKey: 'revenue-and-contracts',
target: 'wiki',
action: 'created',
key: 'knowledge/global/revenue-gross-to-net.md',
key: 'wiki/global/revenue-gross-to-net.md',
},
{
type: 'candidate_action',
unitKey: 'revenue-and-contracts',
target: 'wiki',
action: 'created',
key: 'knowledge/global/discount-expiration.md',
key: 'wiki/global/discount-expiration.md',
},
{
type: 'candidate_action',
@ -709,20 +709,20 @@ function buildReplay(provenance, transcripts) {
key: 'orbit_demo.arr_movements',
},
{ type: 'work_unit_finished', unitKey: 'revenue-and-contracts', status: 'success' },
{ type: 'work_unit_started', unitKey: 'retention-and-segments', skills: ['knowledge_capture', 'sl_capture'], stepBudget: 40 },
{ type: 'work_unit_started', unitKey: 'retention-and-segments', skills: ['wiki_capture', 'sl_capture'], stepBudget: 40 },
{
type: 'candidate_action',
unitKey: 'retention-and-segments',
target: 'wiki',
action: 'created',
key: 'knowledge/global/nrr-retention.md',
key: 'wiki/global/nrr-retention.md',
},
{
type: 'candidate_action',
unitKey: 'retention-and-segments',
target: 'wiki',
action: 'created',
key: 'knowledge/global/segment-classification.md',
key: 'wiki/global/segment-classification.md',
},
{
type: 'candidate_action',
@ -735,7 +735,7 @@ function buildReplay(provenance, transcripts) {
{
type: 'work_unit_started',
unitKey: 'procurement-and-activation',
skills: ['knowledge_capture', 'sl_capture'],
skills: ['wiki_capture', 'sl_capture'],
stepBudget: 40,
},
{
@ -743,14 +743,14 @@ function buildReplay(provenance, transcripts) {
unitKey: 'procurement-and-activation',
target: 'wiki',
action: 'created',
key: 'knowledge/global/activation-policy.md',
key: 'wiki/global/activation-policy.md',
},
{
type: 'candidate_action',
unitKey: 'procurement-and-activation',
target: 'wiki',
action: 'created',
key: 'knowledge/global/procurement-workflows.md',
key: 'wiki/global/procurement-workflows.md',
},
{
type: 'candidate_action',
@ -760,20 +760,20 @@ function buildReplay(provenance, transcripts) {
key: 'orbit_demo.purchase_requests',
},
{ type: 'work_unit_finished', unitKey: 'procurement-and-activation', status: 'success' },
{ type: 'work_unit_started', unitKey: 'support-and-health', skills: ['knowledge_capture', 'sl_capture'], stepBudget: 40 },
{ type: 'work_unit_started', unitKey: 'support-and-health', skills: ['wiki_capture', 'sl_capture'], stepBudget: 40 },
{
type: 'candidate_action',
unitKey: 'support-and-health',
target: 'wiki',
action: 'created',
key: 'knowledge/global/customer-health-scoring.md',
key: 'wiki/global/customer-health-scoring.md',
},
{
type: 'candidate_action',
unitKey: 'support-and-health',
target: 'wiki',
action: 'created',
key: 'knowledge/global/support-escalation.md',
key: 'wiki/global/support-escalation.md',
},
{
type: 'candidate_action',
@ -783,13 +783,13 @@ function buildReplay(provenance, transcripts) {
key: 'orbit_demo.support_tickets',
},
{ type: 'work_unit_finished', unitKey: 'support-and-health', status: 'success' },
{ type: 'work_unit_started', unitKey: 'governance-and-exclusions', skills: ['knowledge_capture'], stepBudget: 40 },
{ type: 'work_unit_started', unitKey: 'governance-and-exclusions', skills: ['wiki_capture'], stepBudget: 40 },
{
type: 'candidate_action',
unitKey: 'governance-and-exclusions',
target: 'wiki',
action: 'created',
key: 'knowledge/global/internal-test-exclusion.md',
key: 'wiki/global/internal-test-exclusion.md',
},
{ type: 'work_unit_finished', unitKey: 'governance-and-exclusions', status: 'success' },
{ type: 'reconciliation_finished', conflictCount: 0, fallbackCount: 0 },
@ -835,7 +835,7 @@ function buildReplay(provenance, transcripts) {
async function writeGeneratedContext(rowCounts) {
for (const page of knowledgePages) {
await writeText(join('knowledge/global', page.file), renderKnowledgePage(page));
await writeText(join('wiki/global', page.file), renderKnowledgePage(page));
}
for (const table of semanticLayerTables) {
@ -908,7 +908,7 @@ async function writeGeneratedContext(rowCounts) {
},
generated: {
semanticLayer: { path: 'semantic-layer/orbit_demo', sourceCount: 6 },
knowledge: { path: 'knowledge/global', pageCount: 10 },
knowledge: { path: 'wiki/global', pageCount: 10 },
links: { path: 'links', linkCount: provenanceLinks.length },
},
});
@ -930,7 +930,7 @@ for (const relativeDir of [
'raw-sources/bi',
'raw-sources/notion',
'semantic-layer/orbit_demo',
'knowledge/global',
'wiki/global',
'links',
'reports',
]) {

152
packages/cli/src/agent-runtime.test.ts
View file

@ -1,152 +0,0 @@
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import {
KTX_AGENT_MAX_ROWS_CAP,
createKtxAgentRuntime,
parseAgentMaxRows,
readAgentJsonFile,
writeAgentJson,
writeAgentJsonError,
} from './agent-runtime.js';
function makeIo() {
let stdout = '';
let stderr = '';
return {
io: {
stdout: { write: (chunk: string) => (stdout += chunk) },
stderr: { write: (chunk: string) => (stderr += chunk) },
},
stdout: () => stdout,
stderr: () => stderr,
};
}
describe('agent runtime helpers', () => {
let tempDir: string;
beforeEach(async () => {
tempDir = await mkdtemp(join(tmpdir(), 'ktx-agent-runtime-'));
});
afterEach(async () => {
await rm(tempDir, { recursive: true, force: true });
});
it('writes JSON success and error envelopes without color or spinners', () => {
const successIo = makeIo();
const errorIo = makeIo();
writeAgentJson(successIo.io, { ok: true });
writeAgentJsonError(errorIo.io, 'missing source', { code: 'NOT_FOUND' });
expect(JSON.parse(successIo.stdout())).toEqual({ ok: true });
expect(successIo.stderr()).toBe('');
expect(JSON.parse(errorIo.stderr())).toEqual({
ok: false,
error: { message: 'missing source', code: 'NOT_FOUND' },
});
expect(errorIo.stdout()).toBe('');
});
it('reads JSON query files as objects', async () => {
const path = join(tempDir, 'query.json');
await writeFile(path, '{"measures":["revenue"],"limit":50}', 'utf-8');
await expect(readAgentJsonFile(path)).resolves.toEqual({ measures: ['revenue'], limit: 50 });
});
it('rejects non-object JSON query files', async () => {
const path = join(tempDir, 'query.json');
await writeFile(path, '["revenue"]', 'utf-8');
await expect(readAgentJsonFile(path)).rejects.toThrow('must contain a JSON object');
});
it('requires positive row limits and enforces the agent cap', () => {
expect(parseAgentMaxRows(100)).toBe(100);
expect(() => parseAgentMaxRows(undefined)).toThrow('maxRows is required');
expect(() => parseAgentMaxRows(0)).toThrow('positive integer');
expect(() => parseAgentMaxRows(KTX_AGENT_MAX_ROWS_CAP + 1)).toThrow(String(KTX_AGENT_MAX_ROWS_CAP));
});
it('constructs local context ports with semantic compute and query executor', async () => {
const project = {
projectDir: tempDir,
configPath: join(tempDir, 'ktx.yaml'),
config: { project: 'revenue', connections: {} },
coreConfig: {},
git: {},
fileStore: {},
} as never;
const ports = { knowledge: {}, semanticLayer: {} } as never;
const semanticLayerCompute = { query: vi.fn(), validateSources: vi.fn(), generateSources: vi.fn() };
const queryExecutor = { execute: vi.fn() };
const loadProject = vi.fn(async () => project);
const createContextTools = vi.fn(() => ports);
await expect(
createKtxAgentRuntime(
{ projectDir: tempDir, enableSemanticCompute: true, enableQueryExecution: true },
{
loadProject,
createContextTools,
createSemanticLayerCompute: () => semanticLayerCompute,
createQueryExecutor: () => queryExecutor,
},
),
).resolves.toMatchObject({ project, ports, queryExecutor });
expect(loadProject).toHaveBeenCalledWith({ projectDir: tempDir });
expect(createContextTools).toHaveBeenCalledWith(project, {
semanticLayerCompute,
queryExecutor,
});
});
it('creates managed semantic compute when no test override is injected', async () => {
const project = {
projectDir: tempDir,
configPath: join(tempDir, 'ktx.yaml'),
config: { project: 'revenue', connections: {} },
coreConfig: {},
git: {},
fileStore: {},
} as never;
const ports = { semanticLayer: {} } as never;
const semanticLayerCompute = { query: vi.fn(), validateSources: vi.fn(), generateSources: vi.fn() };
const loadProject = vi.fn(async () => project);
const createContextTools = vi.fn(() => ports);
const createManagedSemanticLayerCompute = vi.fn(async () => semanticLayerCompute);
const { io } = makeIo();
await expect(
createKtxAgentRuntime(
{
projectDir: tempDir,
enableSemanticCompute: true,
enableQueryExecution: false,
cliVersion: '0.2.0',
runtimeInstallPolicy: 'auto',
io,
},
{
loadProject,
createContextTools,
createManagedSemanticLayerCompute,
},
),
).resolves.toMatchObject({ project, ports, semanticLayerCompute });
expect(createManagedSemanticLayerCompute).toHaveBeenCalledWith({
cliVersion: '0.2.0',
installPolicy: 'auto',
io,
});
expect(createContextTools).toHaveBeenCalledWith(project, {
semanticLayerCompute,
});
});
});

109
packages/cli/src/agent-runtime.ts
View file

@ -1,109 +0,0 @@
import { readFile } from 'node:fs/promises';
import { createDefaultLocalQueryExecutor, type KtxSqlQueryExecutorPort } from '@ktx/context/connections';
import type { KtxSemanticLayerComputePort } from '@ktx/context/daemon';
import { createLocalProjectMcpContextPorts, type KtxMcpContextPorts } from '@ktx/context/mcp';
import { type KtxLocalProject, loadKtxProject } from '@ktx/context/project';
import type { KtxCliIo } from './cli-runtime.js';
import {
createManagedPythonSemanticLayerComputePort,
type KtxManagedPythonInstallPolicy,
} from './managed-python-command.js';
export const KTX_AGENT_MAX_ROWS_CAP = 1000;
export interface KtxAgentRuntimeOptions {
projectDir: string;
enableSemanticCompute: boolean;
enableQueryExecution: boolean;
cliVersion?: string;
runtimeInstallPolicy?: KtxManagedPythonInstallPolicy;
io?: KtxCliIo;
}
export interface KtxAgentRuntime {
project: KtxLocalProject;
ports: KtxMcpContextPorts;
semanticLayerCompute?: KtxSemanticLayerComputePort;
queryExecutor?: KtxSqlQueryExecutorPort;
}
export interface KtxAgentRuntimeDeps {
loadProject?: typeof loadKtxProject;
createContextTools?: typeof createLocalProjectMcpContextPorts;
createSemanticLayerCompute?: () => KtxSemanticLayerComputePort;
createManagedSemanticLayerCompute?: typeof createManagedPythonSemanticLayerComputePort;
createQueryExecutor?: () => KtxSqlQueryExecutorPort;
}
export function writeAgentJson(io: KtxCliIo, value: unknown): void {
io.stdout.write(`${JSON.stringify(value, null, 2)}\n`);
}
export function writeAgentJsonError(
io: KtxCliIo,
message: string,
detail: Record<string, unknown> = {},
): void {
io.stderr.write(`${JSON.stringify({ ok: false, error: { message, ...detail } }, null, 2)}\n`);
}
export async function readAgentJsonFile(path: string): Promise<Record<string, unknown>> {
const parsed = JSON.parse(await readFile(path, 'utf-8')) as unknown;
if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
throw new Error(`${path} must contain a JSON object.`);
}
return parsed as Record<string, unknown>;
}
export function parseAgentMaxRows(value: number | undefined): number {
if (!Number.isInteger(value) || value === undefined || value <= 0) {
throw new Error('maxRows is required and must be a positive integer.');
}
if (value > KTX_AGENT_MAX_ROWS_CAP) {
throw new Error(`maxRows must be less than or equal to ${KTX_AGENT_MAX_ROWS_CAP}.`);
}
return value;
}
async function createAgentSemanticLayerCompute(
options: KtxAgentRuntimeOptions,
deps: KtxAgentRuntimeDeps,
): Promise<KtxSemanticLayerComputePort | undefined> {
if (!options.enableSemanticCompute) {
return undefined;
}
if (deps.createSemanticLayerCompute) {
return deps.createSemanticLayerCompute();
}
if (!options.cliVersion || !options.runtimeInstallPolicy || !options.io) {
throw new Error('Managed Python semantic compute requires cliVersion, runtimeInstallPolicy, and io.');
}
const createManagedSemanticLayerCompute =
deps.createManagedSemanticLayerCompute ?? createManagedPythonSemanticLayerComputePort;
return createManagedSemanticLayerCompute({
cliVersion: options.cliVersion,
installPolicy: options.runtimeInstallPolicy,
io: options.io,
});
}
export async function createKtxAgentRuntime(
options: KtxAgentRuntimeOptions,
deps: KtxAgentRuntimeDeps = {},
): Promise<KtxAgentRuntime> {
const project = await (deps.loadProject ?? loadKtxProject)({ projectDir: options.projectDir });
const semanticLayerCompute = await createAgentSemanticLayerCompute(options, deps);
const queryExecutor = options.enableQueryExecution
? (deps.createQueryExecutor ?? createDefaultLocalQueryExecutor)()
: undefined;
const ports = (deps.createContextTools ?? createLocalProjectMcpContextPorts)(project, {
...(semanticLayerCompute ? { semanticLayerCompute } : {}),
...(queryExecutor ? { queryExecutor } : {}),
});
return {
project,
ports,
...(semanticLayerCompute ? { semanticLayerCompute } : {}),
...(queryExecutor ? { queryExecutor } : {}),
};
}

51
packages/cli/src/agent-search-readiness.test.ts
View file

@ -1,51 +0,0 @@
import { describe, expect, it } from 'vitest';
import {
isMissingProjectConfigError,
missingConnectionSlSearchReadiness,
missingProjectSlSearchReadiness,
noConnectionsSlSearchReadiness,
noIndexedSourcesSlSearchReadiness,
} from './agent-search-readiness.js';
describe('agent semantic-layer search readiness guidance', () => {
it('formats missing project guidance with exact recovery commands', () => {
expect(missingProjectSlSearchReadiness('/tmp/ktx-search', 'gross revenue')).toEqual({
code: 'agent_sl_search_missing_project',
message: 'Semantic-layer search needs an initialized KTX project at /tmp/ktx-search.',
nextSteps: [
'ktx setup --project-dir /tmp/ktx-search',
'ktx status --project-dir /tmp/ktx-search',
'ktx ingest <connection>',
'ktx agent sl list --json --query "gross revenue" --project-dir /tmp/ktx-search',
],
});
});
it('formats no-connection and no-index guidance without hiding the project path', () => {
expect(noConnectionsSlSearchReadiness('/tmp/ktx-search', 'revenue')).toMatchObject({
code: 'agent_sl_search_no_connections',
message: 'Semantic-layer search found no configured connections in /tmp/ktx-search.',
});
expect(noIndexedSourcesSlSearchReadiness('/tmp/ktx-search', 'orders')).toMatchObject({
code: 'agent_sl_search_no_indexed_sources',
message: 'Semantic-layer search found no indexed semantic-layer sources in /tmp/ktx-search.',
});
});
it('formats unknown connection guidance', () => {
expect(missingConnectionSlSearchReadiness('/tmp/ktx-search', 'warehouse', 'revenue')).toMatchObject({
code: 'agent_sl_search_unknown_connection',
message: 'Semantic-layer search connection "warehouse" is not configured in /tmp/ktx-search.',
});
});
it('detects missing ktx.yaml read errors', () => {
const error = Object.assign(new Error('ENOENT: no such file or directory'), {
code: 'ENOENT',
path: '/tmp/ktx-search/ktx.yaml',
});
expect(isMissingProjectConfigError(error)).toBe(true);
expect(isMissingProjectConfigError(new Error('other'))).toBe(false);
});
});

94
packages/cli/src/agent-search-readiness.ts
View file

@ -1,94 +0,0 @@
export type KtxAgentSlSearchReadinessCode =
| 'agent_sl_search_missing_project'
| 'agent_sl_search_no_connections'
| 'agent_sl_search_unknown_connection'
| 'agent_sl_search_no_indexed_sources';
export interface KtxAgentSlSearchReadinessDetail {
code: KtxAgentSlSearchReadinessCode;
message: string;
nextSteps: string[];
}
function queryForCommand(query: string | undefined): string {
const trimmed = query?.trim();
return trimmed && trimmed.length > 0 ? trimmed : 'revenue';
}
function projectSearchCommand(projectDir: string, query: string | undefined): string {
return `ktx agent sl list --json --query ${JSON.stringify(queryForCommand(query))} --project-dir ${projectDir}`;
}
function baseNextSteps(projectDir: string, query: string | undefined): string[] {
return [
`ktx setup --project-dir ${projectDir}`,
`ktx status --project-dir ${projectDir}`,
'ktx ingest <connection>',
projectSearchCommand(projectDir, query),
];
}
export function missingProjectSlSearchReadiness(
projectDir: string,
query: string | undefined,
): KtxAgentSlSearchReadinessDetail {
return {
code: 'agent_sl_search_missing_project',
message: `Semantic-layer search needs an initialized KTX project at ${projectDir}.`,
nextSteps: baseNextSteps(projectDir, query),
};
}
export function noConnectionsSlSearchReadiness(
projectDir: string,
query: string | undefined,
): KtxAgentSlSearchReadinessDetail {
return {
code: 'agent_sl_search_no_connections',
message: `Semantic-layer search found no configured connections in ${projectDir}.`,
nextSteps: baseNextSteps(projectDir, query),
};
}
export function missingConnectionSlSearchReadiness(
projectDir: string,
connectionId: string,
query: string | undefined,
): KtxAgentSlSearchReadinessDetail {
return {
code: 'agent_sl_search_unknown_connection',
message: `Semantic-layer search connection "${connectionId}" is not configured in ${projectDir}.`,
nextSteps: baseNextSteps(projectDir, query),
};
}
export function noIndexedSourcesSlSearchReadiness(
projectDir: string,
query: string | undefined,
): KtxAgentSlSearchReadinessDetail {
return {
code: 'agent_sl_search_no_indexed_sources',
message: `Semantic-layer search found no indexed semantic-layer sources in ${projectDir}.`,
nextSteps: baseNextSteps(projectDir, query),
};
}
function errorCode(error: unknown): string | undefined {
if (typeof error !== 'object' || error === null || !('code' in error)) {
return undefined;
}
const code = (error as { code?: unknown }).code;
return typeof code === 'string' ? code : undefined;
}
function errorPath(error: unknown): string | undefined {
if (typeof error !== 'object' || error === null || !('path' in error)) {
return undefined;
}
const path = (error as { path?: unknown }).path;
return typeof path === 'string' ? path : undefined;
}
export function isMissingProjectConfigError(error: unknown): boolean {
return errorCode(error) === 'ENOENT' && (errorPath(error)?.endsWith('ktx.yaml') ?? false);
}

428
packages/cli/src/agent.test.ts
View file

@ -1,428 +0,0 @@
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { buildDefaultKtxProjectConfig } from '@ktx/context/project';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { runKtxAgent } from './agent.js';
import type { KtxAgentRuntime } from './agent-runtime.js';
function makeIo() {
let stdout = '';
let stderr = '';
return {
io: {
stdout: { write: (chunk: string) => (stdout += chunk) },
stderr: { write: (chunk: string) => (stderr += chunk) },
},
stdout: () => stdout,
stderr: () => stderr,
};
}
function runtime(overrides: Record<string, unknown> = {}): KtxAgentRuntime {
const config = buildDefaultKtxProjectConfig('revenue');
return {
project: {
projectDir: '/tmp/revenue',
configPath: '/tmp/revenue/ktx.yaml',
config: {
...config,
connections: {
warehouse: { driver: 'sqlite', path: 'warehouse.sqlite', readonly: true as const },
},
},
coreConfig: {} as KtxAgentRuntime['project']['coreConfig'],
git: {} as KtxAgentRuntime['project']['git'],
fileStore: {} as KtxAgentRuntime['project']['fileStore'],
},
ports: {
connections: { list: vi.fn(async () => [{ id: 'warehouse', name: 'warehouse', connectionType: 'sqlite' }]) },
semanticLayer: {
listSources: vi.fn(async () => ({
sources: [
{
connectionId: 'warehouse',
connectionName: 'warehouse',
name: 'orders',
columnCount: 2,
measureCount: 1,
joinCount: 0,
},
],
totalSources: 1,
})),
readSource: vi.fn(async () => ({ sourceName: 'orders', yaml: 'name: orders\n' })),
writeSource: vi.fn(async () => ({ success: true, sourceName: 'orders' })),
validate: vi.fn(async () => ({ success: true, errors: [], warnings: [] })),
query: vi.fn(async () => ({ sql: 'select 1', headers: ['x'], rows: [[1]], totalRows: 1, plan: {} })),
},
knowledge: {
search: vi.fn(async () => ({
results: [
{
key: 'page-1',
path: 'knowledge/global/page-1.md',
scope: 'GLOBAL' as const,
summary: 'Revenue logic',
score: 0.9,
matchReasons: ['lexical' as const],
},
],
totalFound: 1,
})),
read: vi.fn(async () => ({
key: 'page-1',
scope: 'GLOBAL' as const,
summary: 'Revenue logic',
content: 'Use net revenue.',
})),
write: vi.fn(async () => ({ success: true, key: 'page-1', action: 'created' as const })),
},
},
queryExecutor: {
execute: vi.fn(async () => ({ headers: ['x'], rows: [[1]], totalRows: 1, command: 'SELECT', rowCount: 1 })),
},
...overrides,
};
}
function runtimeWithoutConnections(): KtxAgentRuntime {
const base = runtime();
return {
...base,
project: {
...base.project,
config: {
...base.project.config,
connections: {},
},
},
ports: {
...base.ports,
semanticLayer: {
...base.ports.semanticLayer!,
listSources: vi.fn(async () => ({ sources: [], totalSources: 0 })),
},
},
};
}
describe('runKtxAgent', () => {
let tempDir: string;
beforeEach(async () => {
tempDir = await mkdtemp(join(tmpdir(), 'ktx-agent-'));
});
afterEach(async () => {
await rm(tempDir, { recursive: true, force: true });
});
it('prints tool discovery with every stable command', async () => {
const io = makeIo();
await expect(runKtxAgent({ command: 'tools', projectDir: tempDir, json: true }, io.io)).resolves.toBe(0);
const body = JSON.parse(io.stdout());
expect(body.projectDir).toBe(tempDir);
expect(body.tools.map((tool: { name: string }) => tool.name)).toEqual([
'context',
'sl.list',
'sl.read',
'sl.query',
'wiki.search',
'wiki.read',
'sql.execute',
]);
expect(io.stderr()).toBe('');
});
it('prints project context from setup status, connections, and SL summaries', async () => {
const io = makeIo();
const createRuntime = vi.fn(async () => runtime());
const readSetupStatus = vi.fn(async () => ({ project: { path: tempDir, ready: true }, agents: [] }));
await expect(
runKtxAgent({ command: 'context', projectDir: tempDir, json: true }, io.io, { createRuntime, readSetupStatus }),
).resolves.toBe(0);
expect(JSON.parse(io.stdout())).toMatchObject({
projectDir: tempDir,
status: { project: { ready: true } },
connections: [{ id: 'warehouse' }],
semanticLayer: { totalSources: 1 },
});
});
it('dispatches SL list, SL read, wiki search, and wiki read through local ports', async () => {
for (const args of [
{ command: 'sl-list' as const, projectDir: tempDir, json: true as const, connectionId: 'warehouse' },
{
command: 'sl-read' as const,
projectDir: tempDir,
json: true as const,
connectionId: 'warehouse',
sourceName: 'orders',
},
{ command: 'wiki-search' as const, projectDir: tempDir, json: true as const, query: 'revenue', limit: 10 },
{ command: 'wiki-read' as const, projectDir: tempDir, json: true as const, pageId: 'page-1' },
]) {
const io = makeIo();
await expect(runKtxAgent(args, io.io, { createRuntime: async () => runtime() })).resolves.toBe(0);
expect(JSON.parse(io.stdout())).toBeTruthy();
expect(io.stderr()).toBe('');
}
});
it('prints wiki hybrid search metadata from the hidden agent wiki search command', async () => {
const fakeRuntime = runtime();
const knowledge = fakeRuntime.ports.knowledge;
if (!knowledge) {
throw new Error('Expected runtime knowledge port');
}
fakeRuntime.ports.knowledge = {
...knowledge,
search: vi.fn(async () => ({
results: [
{
key: 'metrics-revenue',
path: 'knowledge/global/metrics-revenue.md',
scope: 'GLOBAL' as const,
summary: 'Revenue metric definition',
score: 0.02459016393442623,
matchReasons: ['lexical' as const, 'token' as const],
},
],
totalFound: 1,
})),
};
const io = makeIo();
await expect(
runKtxAgent({ command: 'wiki-search', projectDir: tempDir, json: true, query: 'paid order', limit: 5 }, io.io, {
createRuntime: async () => fakeRuntime,
}),
).resolves.toBe(0);
expect(JSON.parse(io.stdout())).toEqual({
results: [
expect.objectContaining({
key: 'metrics-revenue',
path: 'knowledge/global/metrics-revenue.md',
matchReasons: ['lexical', 'token'],
}),
],
totalFound: 1,
});
});
it('executes SL queries from a JSON query file', async () => {
const queryFile = join(tempDir, 'sl-query.json');
const io = makeIo();
await writeFile(queryFile, '{"measures":["total_revenue"],"dimensions":[]}', 'utf-8');
await expect(
runKtxAgent(
{
command: 'sl-query',
projectDir: tempDir,
json: true,
connectionId: 'warehouse',
queryFile,
execute: true,
maxRows: 100,
cliVersion: '0.2.0',
runtimeInstallPolicy: 'never',
},
io.io,
{ createRuntime: async () => runtime() },
),
).resolves.toBe(0);
expect(JSON.parse(io.stdout())).toMatchObject({ sql: 'select 1', rows: [[1]] });
});
it('passes managed runtime options into default SL query runtime creation', async () => {
const queryFile = join(tempDir, 'sl-query.json');
const io = makeIo();
const createRuntime = vi.fn(async () => runtime());
await writeFile(queryFile, '{"measures":["total_revenue"],"dimensions":[]}', 'utf-8');
await expect(
runKtxAgent(
{
command: 'sl-query',
projectDir: tempDir,
json: true,
connectionId: 'warehouse',
queryFile,
execute: false,
cliVersion: '0.2.0',
runtimeInstallPolicy: 'auto',
},
io.io,
{ createRuntime },
),
).resolves.toBe(0);
expect(createRuntime).toHaveBeenCalledWith({
projectDir: tempDir,
enableSemanticCompute: true,
enableQueryExecution: false,
cliVersion: '0.2.0',
runtimeInstallPolicy: 'auto',
io: io.io,
});
});
it('executes read-only SQL from a SQL file with an explicit row limit', async () => {
const sqlFile = join(tempDir, 'query.sql');
const fakeRuntime = runtime();
const io = makeIo();
await writeFile(sqlFile, 'select 1', 'utf-8');
await expect(
runKtxAgent(
{
command: 'sql-execute',
projectDir: tempDir,
json: true,
connectionId: 'warehouse',
sqlFile,
maxRows: 100,
},
io.io,
{ createRuntime: async () => fakeRuntime as never },
),
).resolves.toBe(0);
expect(fakeRuntime.queryExecutor?.execute).toHaveBeenCalledWith({
connectionId: 'warehouse',
projectDir: '/tmp/revenue',
connection: { driver: 'sqlite', path: 'warehouse.sqlite', readonly: true },
sql: 'select 1',
maxRows: 100,
});
});
it('prints guided JSON when semantic-layer search runs outside a project', async () => {
const io = makeIo();
const missingProjectError = Object.assign(new Error('ENOENT: no such file or directory'), {
code: 'ENOENT',
path: join(tempDir, 'ktx.yaml'),
});
await expect(
runKtxAgent(
{ command: 'sl-list', projectDir: tempDir, json: true, query: 'gross revenue' },
io.io,
{ createRuntime: vi.fn(async () => Promise.reject(missingProjectError)) },
),
).resolves.toBe(1);
expect(JSON.parse(io.stderr())).toEqual({
ok: false,
error: {
code: 'agent_sl_search_missing_project',
message: `Semantic-layer search needs an initialized KTX project at ${tempDir}.`,
nextSteps: [
`ktx setup --project-dir ${tempDir}`,
`ktx status --project-dir ${tempDir}`,
'ktx ingest <connection>',
`ktx agent sl list --json --query "gross revenue" --project-dir ${tempDir}`,
],
},
});
expect(io.stdout()).toBe('');
});
it('prints guided JSON when semantic-layer search has no configured connections', async () => {
const io = makeIo();
await expect(
runKtxAgent(
{ command: 'sl-list', projectDir: tempDir, json: true, query: 'revenue' },
io.io,
{ createRuntime: async () => runtimeWithoutConnections() },
),
).resolves.toBe(1);
expect(JSON.parse(io.stderr())).toMatchObject({
ok: false,
error: {
code: 'agent_sl_search_no_connections',
message: `Semantic-layer search found no configured connections in ${tempDir}.`,
nextSteps: [
`ktx setup --project-dir ${tempDir}`,
`ktx status --project-dir ${tempDir}`,
'ktx ingest <connection>',
`ktx agent sl list --json --query "revenue" --project-dir ${tempDir}`,
],
},
});
});
it('prints guided JSON when semantic-layer search asks for an unknown connection', async () => {
const io = makeIo();
await expect(
runKtxAgent(
{ command: 'sl-list', projectDir: tempDir, json: true, connectionId: 'missing', query: 'revenue' },
io.io,
{ createRuntime: async () => runtime() },
),
).resolves.toBe(1);
expect(JSON.parse(io.stderr())).toMatchObject({
ok: false,
error: {
code: 'agent_sl_search_unknown_connection',
message: `Semantic-layer search connection "missing" is not configured in ${tempDir}.`,
},
});
});
it('prints guided JSON when semantic-layer search has no indexed sources', async () => {
const fakeRuntime = runtime();
const semanticLayer = fakeRuntime.ports.semanticLayer!;
fakeRuntime.ports.semanticLayer = {
...semanticLayer,
listSources: vi.fn(async () => ({ sources: [], totalSources: 0 })),
};
const io = makeIo();
await expect(
runKtxAgent(
{ command: 'sl-list', projectDir: tempDir, json: true, connectionId: 'warehouse', query: 'revenue' },
io.io,
{ createRuntime: async () => fakeRuntime },
),
).resolves.toBe(1);
expect(JSON.parse(io.stderr())).toMatchObject({
ok: false,
error: {
code: 'agent_sl_search_no_indexed_sources',
message: `Semantic-layer search found no indexed semantic-layer sources in ${tempDir}.`,
},
});
});
it('returns JSON errors when required ports or records are missing', async () => {
const io = makeIo();
await expect(
runKtxAgent({ command: 'wiki-read', projectDir: tempDir, json: true, pageId: 'missing' }, io.io, {
createRuntime: async () =>
runtime({
ports: { knowledge: { read: vi.fn(async () => null) } },
}) as never,
}),
).resolves.toBe(1);
expect(JSON.parse(io.stderr())).toMatchObject({
ok: false,
error: { message: expect.stringContaining('missing') },
});
});
});

219
packages/cli/src/agent.ts
View file

@ -1,219 +0,0 @@
import { readFile } from 'node:fs/promises';
import type { KtxCliIo } from './cli-runtime.js';
import {
createKtxAgentRuntime,
parseAgentMaxRows,
readAgentJsonFile,
writeAgentJson,
writeAgentJsonError,
type KtxAgentRuntime,
type KtxAgentRuntimeDeps,
} from './agent-runtime.js';
import {
isMissingProjectConfigError,
missingConnectionSlSearchReadiness,
missingProjectSlSearchReadiness,
noConnectionsSlSearchReadiness,
noIndexedSourcesSlSearchReadiness,
type KtxAgentSlSearchReadinessDetail,
} from './agent-search-readiness.js';
import type { KtxManagedPythonInstallPolicy } from './managed-python-command.js';
import { readKtxSetupStatus, type KtxSetupStatus } from './setup.js';
export type KtxAgentArgs =
| { command: 'tools'; projectDir: string; json: true }
| { command: 'context'; projectDir: string; json: true }
| { command: 'sl-list'; projectDir: string; json: true; connectionId?: string; query?: string }
| { command: 'sl-read'; projectDir: string; json: true; connectionId?: string; sourceName: string }
| {
command: 'sl-query';
projectDir: string;
json: true;
connectionId: string;
queryFile: string;
execute: boolean;
maxRows?: number;
cliVersion: string;
runtimeInstallPolicy: KtxManagedPythonInstallPolicy;
}
| { command: 'wiki-search'; projectDir: string; json: true; query: string; limit: number }
| { command: 'wiki-read'; projectDir: string; json: true; pageId: string }
| { command: 'sql-execute'; projectDir: string; json: true; connectionId: string; sqlFile: string; maxRows?: number };
export interface KtxAgentDeps extends KtxAgentRuntimeDeps {
createRuntime?: (options: {
projectDir: string;
enableSemanticCompute: boolean;
enableQueryExecution: boolean;
cliVersion?: string;
runtimeInstallPolicy?: KtxManagedPythonInstallPolicy;
io?: KtxCliIo;
}) => Promise<KtxAgentRuntime>;
readSetupStatus?: (
projectDir: string,
) => Promise<KtxSetupStatus | { project: { path?: string; ready: boolean }; agents: unknown[] }>;
}
const AGENT_TOOLS = [
{ name: 'context', command: 'ktx agent context --json' },
{ name: 'sl.list', command: 'ktx agent sl list --json [--connection-id <id>] [--query <text>]' },
{ name: 'sl.read', command: 'ktx agent sl read <sourceName> --json [--connection-id <id>]' },
{
name: 'sl.query',
command: 'ktx agent sl query --json --connection-id <id> --query-file <path> --execute --max-rows 100',
},
{ name: 'wiki.search', command: 'ktx agent wiki search <query> --json [--limit 10]' },
{ name: 'wiki.read', command: 'ktx agent wiki read <pageId> --json' },
{
name: 'sql.execute',
command: 'ktx agent sql execute --json --connection-id <id> --sql-file <path> --max-rows 100',
},
] as const;
function writeAgentSlSearchReadinessError(io: KtxCliIo, detail: KtxAgentSlSearchReadinessDetail): void {
writeAgentJsonError(io, detail.message, { code: detail.code, nextSteps: detail.nextSteps });
}
async function runtimeFor(args: KtxAgentArgs, deps: KtxAgentDeps, io: KtxCliIo): Promise<KtxAgentRuntime> {
const needsSemanticCompute = args.command === 'sl-query';
const needsQueryExecution = args.command === 'sql-execute' || (args.command === 'sl-query' && args.execute);
const runtimeOptions = {
projectDir: args.projectDir,
enableSemanticCompute: needsSemanticCompute,
enableQueryExecution: needsQueryExecution,
...(args.command === 'sl-query'
? {
cliVersion: args.cliVersion,
runtimeInstallPolicy: args.runtimeInstallPolicy,
io,
}
: {}),
};
return deps.createRuntime ? deps.createRuntime(runtimeOptions) : createKtxAgentRuntime(runtimeOptions, deps);
}
function connectionIdForSource(runtime: KtxAgentRuntime, requested: string | undefined): string {
if (requested) return requested;
const ids = Object.keys(runtime.project.config.connections ?? {});
if (ids.length === 1) return ids[0] as string;
throw new Error('Use --connection-id when the project has zero or multiple connections.');
}
export async function runKtxAgent(args: KtxAgentArgs, io: KtxCliIo, deps: KtxAgentDeps = {}): Promise<number> {
try {
if (args.command === 'tools') {
writeAgentJson(io, { projectDir: args.projectDir, tools: AGENT_TOOLS });
return 0;
}
const runtime = await runtimeFor(args, deps, io);
if (args.command === 'context') {
const [status, connections, semanticLayer] = await Promise.all([
(deps.readSetupStatus ?? readKtxSetupStatus)(args.projectDir),
runtime.ports.connections?.list() ?? [],
runtime.ports.semanticLayer?.listSources({}) ?? { sources: [], totalSources: 0 },
]);
writeAgentJson(io, { projectDir: args.projectDir, status, connections, semanticLayer, tools: AGENT_TOOLS });
return 0;
}
if (args.command === 'sl-list') {
const semanticLayer = runtime.ports.semanticLayer;
if (!semanticLayer) throw new Error('Semantic-layer tools are not available for this project.');
if (args.query) {
const connectionIds = Object.keys(runtime.project.config.connections ?? {});
if (args.connectionId && !runtime.project.config.connections[args.connectionId]) {
writeAgentSlSearchReadinessError(
io,
missingConnectionSlSearchReadiness(args.projectDir, args.connectionId, args.query),
);
return 1;
}
if (connectionIds.length === 0) {
writeAgentSlSearchReadinessError(io, noConnectionsSlSearchReadiness(args.projectDir, args.query));
return 1;
}
}
const listed = await semanticLayer.listSources({ connectionId: args.connectionId, query: args.query });
if (args.query && listed.sources.length === 0) {
const allSources = await semanticLayer.listSources({ connectionId: args.connectionId });
if (allSources.totalSources === 0) {
writeAgentSlSearchReadinessError(io, noIndexedSourcesSlSearchReadiness(args.projectDir, args.query));
return 1;
}
}
writeAgentJson(io, listed);
return 0;
}
if (args.command === 'sl-read') {
const semanticLayer = runtime.ports.semanticLayer;
if (!semanticLayer) throw new Error('Semantic-layer tools are not available for this project.');
const source = await semanticLayer.readSource({
connectionId: connectionIdForSource(runtime, args.connectionId),
sourceName: args.sourceName,
});
if (!source) throw new Error(`Semantic-layer source "${args.sourceName}" was not found.`);
writeAgentJson(io, source);
return 0;
}
if (args.command === 'sl-query') {
const semanticLayer = runtime.ports.semanticLayer;
if (!semanticLayer) throw new Error('Semantic-layer tools are not available for this project.');
const query = await readAgentJsonFile(args.queryFile);
const maxRows = args.execute ? parseAgentMaxRows(args.maxRows) : args.maxRows;
writeAgentJson(
io,
await semanticLayer.query({
connectionId: args.connectionId,
query: { ...query, ...(maxRows !== undefined ? { limit: maxRows } : {}) } as never,
}),
);
return 0;
}
if (args.command === 'wiki-search') {
const knowledge = runtime.ports.knowledge;
if (!knowledge) throw new Error('Wiki tools are not available for this project.');
writeAgentJson(io, await knowledge.search({ userId: 'agent', query: args.query, limit: args.limit }));
return 0;
}
if (args.command === 'wiki-read') {
const knowledge = runtime.ports.knowledge;
if (!knowledge) throw new Error('Wiki tools are not available for this project.');
const page = await knowledge.read({ userId: 'agent', key: args.pageId });
if (!page) throw new Error(`Wiki page "${args.pageId}" was not found.`);
writeAgentJson(io, page);
return 0;
}
const queryExecutor = runtime.queryExecutor;
if (!queryExecutor) throw new Error('SQL execution is not available for this project.');
const connection = runtime.project.config.connections[args.connectionId];
if (!connection) throw new Error(`Connection "${args.connectionId}" was not found.`);
const maxRows = parseAgentMaxRows(args.maxRows);
writeAgentJson(
io,
await queryExecutor.execute({
connectionId: args.connectionId,
projectDir: runtime.project.projectDir,
connection,
sql: await readFile(args.sqlFile, 'utf-8'),
maxRows,
}),
);
return 0;
} catch (error) {
if (args.command === 'sl-list' && args.query && isMissingProjectConfigError(error)) {
writeAgentSlSearchReadinessError(io, missingProjectSlSearchReadiness(args.projectDir, args.query));
return 1;
}
writeAgentJsonError(io, error instanceof Error ? error.message : String(error));
return 1;
}
}

1
packages/cli/src/clack.ts
View file

@ -2,6 +2,7 @@ import { cancel, confirm, isCancel, log, spinner } from '@clack/prompts';
export interface KtxCliSpinner {
start(message: string): void;
message(message: string): void;
stop(message: string): void;
error(message: string): void;
}

29
packages/cli/src/cli-program.ts
View file

@ -1,9 +1,9 @@
import { Command, InvalidArgumentError } from '@commander-js/extra-typings';
import type { KtxCliDeps, KtxCliIo, KtxCliPackageInfo } from './cli-runtime.js';
import { registerAgentCommands } from './commands/agent-commands.js';
import { registerConnectionCommands } from './commands/connection-commands.js';
import { registerIngestCommands } from './commands/ingest-commands.js';
import { registerWikiCommands } from './commands/knowledge-commands.js';
import { registerPublicIngestCommands } from './commands/public-ingest-commands.js';
import { registerScanCommands } from './commands/scan-commands.js';
import { registerSetupCommands } from './commands/setup-commands.js';
import { registerSlCommands } from './commands/sl-commands.js';
import { registerStatusCommands } from './commands/status-commands.js';
@ -53,7 +53,7 @@ type CommandPathNode = CommandWithGlobalOptions & {
parent?: CommandPathNode | null;
};
const PROJECT_AWARE_ROOT_COMMANDS = new Set(['setup', 'connection', 'ingest', 'wiki', 'sl', 'status']);
const PROJECT_AWARE_ROOT_COMMANDS = new Set(['setup', 'connection', 'ingest', 'wiki', 'sl', 'status', 'scan']);
export interface CommandWithGlobalOptions {
opts: () => object;
@ -151,7 +151,7 @@ function isProjectAwareCommand(path: string[]): boolean {
const rootCommand = path[1];
if (rootCommand === 'dev') {
return path[2] !== undefined && path[2] !== 'completion' && path[2] !== 'runtime';
return path[2] !== undefined && path[2] !== 'runtime';
}
return rootCommand !== undefined && PROJECT_AWARE_ROOT_COMMANDS.has(rootCommand);
}
@ -162,6 +162,10 @@ function shouldSuppressProjectDirLine(path: string[], options: Record<string, un
return true;
}
if (commandPathKey === 'ktx setup') {
return true;
}
if (
commandPathKey === 'ktx status' &&
typeof options.projectDir !== 'string' &&
@ -176,14 +180,8 @@ function shouldSuppressProjectDirLine(path: string[], options: Record<string, un
}
if (commandPathKey === 'ktx ingest watch') {
return options.json !== true;
}
if (commandPathKey === 'ktx dev ingest watch') {
return options.json !== true && options.plain !== true;
}
if (commandPathKey === 'ktx connection notion pick') {
return options.input !== false;
}
const demoIndex = path.indexOf('demo');
if (demoIndex >= 0) {
const demoCommand = path[demoIndex + 1];
@ -222,7 +220,7 @@ export function resolveCommandProjectDirOverride(command: CommandWithGlobalOptio
function createBaseProgram(info: KtxCliPackageInfo, io: KtxCliIo): Command {
return new Command()
.name('ktx')
.description('Standalone KTX developer CLI')
.description('KTX data agent context layer CLI')
.option('--project-dir <path>', 'KTX project directory (default: KTX_PROJECT_DIR, nearest ktx.yaml, or cwd)')
.option('--debug', 'Enable diagnostic logging to stderr')
.version(`${info.name} ${info.version}`, '-v, --version', 'Show CLI version')
@ -230,7 +228,7 @@ function createBaseProgram(info: KtxCliPackageInfo, io: KtxCliIo): Command {
.configureHelp({ showGlobalOptions: true })
.addHelpText(
'after',
'\nAdvanced:\n ktx dev Low-level diagnostics, scans, adapter commands, and mapping tools.\n',
'\nAdvanced:\n ktx dev Low-level project initialization and runtime management.\n',
)
.showHelpAfterError()
.exitOverride()
@ -315,11 +313,14 @@ export function buildKtxProgram(options: BuildKtxProgramOptions): Command {
registerSetupCommands(program, context);
registerConnectionCommands(program, context);
registerPublicIngestCommands(program, context);
registerIngestCommands(program, context, {
runIngestWithProgress: async (ingestArgs, ingestIo, ingestDeps, defaultRunIngest) =>
await (ingestDeps.ingest ?? defaultRunIngest)(ingestArgs, ingestIo),
});
registerScanCommands(program, context);
registerWikiCommands(program, context);
registerSlCommands(program, context);
registerStatusCommands(program, context);
registerAgentCommands(program, context);
registerDevCommands(program, context);
return program;

8
packages/cli/src/cli-runtime.ts
View file

@ -1,13 +1,9 @@
import { createRequire } from 'node:module';
import type { KtxConnectionMetabaseSetupArgs } from './commands/connection-metabase-setup.js';
import type { KtxConnectionNotionArgs } from './commands/connection-notion.js';
import type { KtxAgentArgs } from './agent.js';
import type { KtxConnectionArgs } from './connection.js';
import type { KtxDoctorArgs } from './doctor.js';
import type { KtxIngestArgs } from './ingest.js';
import type { KtxKnowledgeArgs } from './knowledge.js';
import type { KtxPublicIngestArgs } from './public-ingest.js';
import type { KtxRuntimeArgs } from './runtime.js';
import type { KtxScanArgs } from './scan.js';
import type { KtxSetupArgs } from './setup.js';
@ -31,13 +27,9 @@ export interface KtxCliIo {
export interface KtxCliDeps {
setup?: (args: KtxSetupArgs, io: KtxCliIo) => Promise<number>;
agent?: (args: KtxAgentArgs, io: KtxCliIo) => Promise<number>;
connection?: (args: KtxConnectionArgs, io: KtxCliIo) => Promise<number>;
connectionNotion?: (args: KtxConnectionNotionArgs, io: KtxCliIo) => Promise<number>;
connectionMetabaseSetup?: (args: KtxConnectionMetabaseSetupArgs, io: KtxCliIo) => Promise<number>;
doctor?: (args: KtxDoctorArgs, io: KtxCliIo) => Promise<number>;
ingest?: (args: KtxIngestArgs, io: KtxCliIo) => Promise<number>;
publicIngest?: (args: KtxPublicIngestArgs, io: KtxCliIo) => Promise<number>;
runtime?: (args: KtxRuntimeArgs, io: KtxCliIo) => Promise<number>;
scan?: (args: KtxScanArgs, io: KtxCliIo) => Promise<number>;
knowledge?: (args: KtxKnowledgeArgs, io: KtxCliIo) => Promise<number>;

63
packages/cli/src/command-schemas.ts
View file

@ -1,33 +1,8 @@
import { z } from 'zod';
const projectDirSchema = z.string().min(1);
const safeConnectionIdSchema = z.string().regex(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/, 'Unsafe connection id');
const stringArraySchema = z.array(z.string());
export const connectionAddCommandSchema = z.object({
command: z.literal('add'),
projectDir: projectDirSchema,
driver: z.string().min(1),
connectionId: safeConnectionIdSchema,
url: z.string().optional(),
schemas: stringArraySchema,
readonly: z.boolean(),
force: z.boolean(),
allowLiteralCredentials: z.boolean(),
notion: z
.object({
authTokenRef: z.string().min(1),
crawlMode: z.enum(['all_accessible', 'selected_roots']),
rootPageIds: stringArraySchema,
rootDatabaseIds: stringArraySchema,
rootDataSourceIds: stringArraySchema,
maxPagesPerRun: z.number().int().positive().optional(),
maxKnowledgeCreatesPerRun: z.number().int().nonnegative().optional(),
maxKnowledgeUpdatesPerRun: z.number().int().nonnegative().optional(),
})
.optional(),
});
export const wikiWriteCommandSchema = z.object({
command: z.literal('write'),
projectDir: projectDirSchema,
@ -53,35 +28,21 @@ export const slQueryCommandSchema = z.object({
command: z.literal('query'),
projectDir: projectDirSchema,
connectionId: z.string().min(1).optional(),
query: z.object({
measures: z.array(z.string().min(1)).min(1),
dimensions: stringArraySchema,
filters: stringArraySchema.optional(),
segments: stringArraySchema.optional(),
order_by: z.array(orderBySchema).optional(),
limit: z.number().int().positive().optional(),
include_empty: z.literal(true).optional(),
}),
query: z
.object({
measures: z.array(z.string().min(1)).min(1),
dimensions: stringArraySchema,
filters: stringArraySchema.optional(),
segments: stringArraySchema.optional(),
order_by: z.array(orderBySchema).optional(),
limit: z.number().int().positive().optional(),
include_empty: z.literal(true).optional(),
})
.optional(),
queryFile: z.string().min(1).optional(),
format: z.enum(['json', 'sql']),
execute: z.boolean(),
cliVersion: z.string().min(1),
runtimeInstallPolicy: z.enum(['prompt', 'auto', 'never']),
maxRows: z.number().int().positive().optional(),
});
export const publicIngestRunCommandSchema = z.object({
command: z.literal('run'),
projectDir: projectDirSchema,
targetConnectionId: safeConnectionIdSchema.optional(),
all: z.boolean(),
json: z.boolean(),
inputMode: z.enum(['auto', 'disabled']),
});
export const publicIngestReadCommandSchema = z.object({
command: z.enum(['status', 'watch']),
projectDir: projectDirSchema,
runId: z.string().min(1).optional(),
json: z.boolean(),
inputMode: z.enum(['auto', 'disabled']),
});

149
packages/cli/src/commands/agent-commands.ts
View file

@ -1,149 +0,0 @@
import { Option, type Command } from '@commander-js/extra-typings';
import type { KtxAgentArgs } from '../agent.js';
import type { KtxCliCommandContext } from '../cli-program.js';
import { parsePositiveIntegerOption, resolveCommandProjectDir } from '../cli-program.js';
import { runtimeInstallPolicyFromFlags } from '../managed-python-command.js';
async function runAgent(context: KtxCliCommandContext, args: KtxAgentArgs): Promise<void> {
const runner = context.deps.agent ?? (await import('../agent.js')).runKtxAgent;
context.setExitCode(await runner(args, context.io));
}
function jsonOption(): Option {
return new Option('--json', 'Print JSON output').makeOptionMandatory();
}
export function registerAgentCommands(program: Command, context: KtxCliCommandContext): void {
const agent = program
.command('agent', { hidden: true })
.description('Machine-readable KTX commands for coding agents')
.showHelpAfterError();
agent.hook('preAction', (_thisCommand, actionCommand) => {
context.writeDebug?.('agent', actionCommand);
});
agent
.command('tools')
.description('Print available agent-facing KTX tools')
.addOption(jsonOption())
.action(async (_options, command) => {
await runAgent(context, { command: 'tools', projectDir: resolveCommandProjectDir(command), json: true });
});
agent
.command('context')
.description('Print project context for agent planning')
.addOption(jsonOption())
.action(async (_options, command) => {
await runAgent(context, { command: 'context', projectDir: resolveCommandProjectDir(command), json: true });
});
const sl = agent.command('sl').description('Semantic-layer agent commands');
sl.command('list')
.description('List semantic-layer sources')
.addOption(jsonOption())
.option('--connection-id <id>', 'Filter by connection id')
.option('--query <text>', 'Search source names and descriptions')
.action(async (options: { connectionId?: string; query?: string }, command) => {
await runAgent(context, {
command: 'sl-list',
projectDir: resolveCommandProjectDir(command),
json: true,
...(options.connectionId ? { connectionId: options.connectionId } : {}),
...(options.query ? { query: options.query } : {}),
});
});
sl.command('read')
.description('Read one semantic-layer source')
.argument('<sourceName>')
.addOption(jsonOption())
.option('--connection-id <id>', 'Connection id containing the source')
.action(async (sourceName: string, options: { connectionId?: string }, command) => {
await runAgent(context, {
command: 'sl-read',
projectDir: resolveCommandProjectDir(command),
json: true,
sourceName,
...(options.connectionId ? { connectionId: options.connectionId } : {}),
});
});
sl.command('query')
.description('Run a semantic-layer query JSON file')
.addOption(jsonOption())
.requiredOption('--connection-id <id>', 'Connection id for execution')
.requiredOption('--query-file <path>', 'JSON semantic-layer query file')
.option('--execute', 'Execute the compiled query against the connection', false)
.option('--yes', 'Install the managed Python runtime without prompting when required', false)
.option('--no-input', 'Disable interactive managed runtime installation')
.option('--max-rows <number>', 'Maximum rows to return when executing', parsePositiveIntegerOption)
.action(
async (
options: {
connectionId: string;
queryFile: string;
execute: boolean;
maxRows?: number;
yes?: boolean;
input?: boolean;
},
command,
) => {
await runAgent(context, {
command: 'sl-query',
projectDir: resolveCommandProjectDir(command),
json: true,
connectionId: options.connectionId,
queryFile: options.queryFile,
execute: options.execute,
cliVersion: context.packageInfo.version,
runtimeInstallPolicy: runtimeInstallPolicyFromFlags(options),
...(options.maxRows !== undefined ? { maxRows: options.maxRows } : {}),
});
},
);
const wiki = agent.command('wiki').description('KTX wiki agent commands');
wiki
.command('search')
.description('Search KTX wiki pages')
.argument('<query>')
.addOption(jsonOption())
.option('--limit <number>', 'Maximum search results', parsePositiveIntegerOption, 10)
.action(async (query: string, options: { limit: number }, command) => {
await runAgent(context, {
command: 'wiki-search',
projectDir: resolveCommandProjectDir(command),
json: true,
query,
limit: options.limit,
});
});
wiki
.command('read')
.description('Read one KTX wiki page')
.argument('<pageId>')
.addOption(jsonOption())
.action(async (pageId: string, _options, command) => {
await runAgent(context, { command: 'wiki-read', projectDir: resolveCommandProjectDir(command), json: true, pageId });
});
const sql = agent.command('sql').description('Safe SQL execution commands');
sql
.command('execute')
.description('Execute read-only SQL with a row limit')
.addOption(jsonOption())
.requiredOption('--connection-id <id>', 'Connection id for execution')
.requiredOption('--sql-file <path>', 'SQL file to execute')
.requiredOption('--max-rows <number>', 'Maximum rows to return', parsePositiveIntegerOption)
.action(async (options: { connectionId: string; sqlFile: string; maxRows: number }, command) => {
await runAgent(context, {
command: 'sql-execute',
projectDir: resolveCommandProjectDir(command),
json: true,
connectionId: options.connectionId,
sqlFile: options.sqlFile,
maxRows: options.maxRows,
});
});
}

47
packages/cli/src/commands/completion-commands.ts
View file

@ -1,47 +0,0 @@
import type { CommandUnknownOpts } from '@commander-js/extra-typings';
import type { KtxCliCommandContext } from '../cli-program.js';
import { completeCommanderInput, installZshCompletion, zshCompletionScript } from '../completion.js';
export function registerCompletionCommands(
program: CommandUnknownOpts,
context: KtxCliCommandContext,
completionRoot: CommandUnknownOpts = program,
): void {
program
.command('completion')
.description('Generate shell completion scripts')
.command('zsh')
.description('Generate zsh completion script')
.option('--install', 'Install zsh completion into ~/.zfunc and update ~/.zshrc', false)
.action(async (options: { install?: boolean }) => {
if (options.install === true) {
const result = await installZshCompletion();
context.io.stdout.write(`Installed zsh completion: ${result.completionPath}\n`);
context.io.stdout.write(`Updated zsh config: ${result.zshrcPath}\n`);
context.io.stdout.write('Restart your shell or run: source ~/.zshrc\n');
context.setExitCode(0);
return;
}
context.io.stdout.write(zshCompletionScript());
context.setExitCode(0);
});
program
.command('__complete', { hidden: true })
.description('Internal shell completion endpoint')
.requiredOption('--shell <shell>', 'Shell requesting completions')
.requiredOption('--position <position>', 'Current shell word position', (value) => Number(value))
.argument('[words...]', 'Current shell words')
.allowUnknownOption()
.allowExcessArguments()
.action((words: string[], options: { shell: string; position: number }) => {
if (options.shell !== 'zsh') {
context.setExitCode(1);
return;
}
for (const completion of completeCommanderInput(completionRoot, { position: options.position, words })) {
context.io.stdout.write(`${completion}\n`);
}
context.setExitCode(0);
});
}

308
packages/cli/src/commands/connection-commands.ts
View file

@ -1,61 +1,19 @@
import { type Command, InvalidArgumentError, Option } from '@commander-js/extra-typings';
import {
collectOption,
type KtxCliCommandContext,
parseBooleanStringOption,
parseNonEmptyAssignmentOption,
parseNonNegativeIntegerOption,
parsePositiveIntegerOption,
parseSafeConnectionIdOption,
resolveCommandProjectDir,
} from '../cli-program.js';
import { connectionAddCommandSchema } from '../command-schemas.js';
import { type Command } from '@commander-js/extra-typings';
import { type KtxCliCommandContext, resolveCommandProjectDir } from '../cli-program.js';
import type { KtxConnectionArgs } from '../connection.js';
import { profileMark } from '../startup-profile.js';
import type { KtxConnectionMappingArgs } from './connection-mapping.js';
import { registerConnectionMetabaseCommands } from './connection-metabase-commands.js';
import { registerConnectionNotionCommands } from './connection-notion-commands.js';
profileMark('module:commands/connection-commands');
const CRAWL_MODE_CHOICES = ['all_accessible', 'selected_roots'] as const;
const SYNC_MODE_CHOICES = ['ALL', 'ONLY', 'EXCEPT'] as const;
function parseCsvIds(value: string): number[] {
return value
.split(',')
.filter(Boolean)
.map((item) => parsePositiveIntegerOption(item));
}
function parseCsvStrings(value: string): string[] {
return value
.split(',')
.map((item) => item.trim())
.filter(Boolean);
}
function parseMappingFieldOption(value: string): 'databaseMappings' | 'connectionMappings' {
if (value === 'databaseMappings' || value === 'connectionMappings') {
return value;
}
throw new InvalidArgumentError('must be databaseMappings or connectionMappings');
}
async function runConnectionArgs(context: KtxCliCommandContext, args: KtxConnectionArgs): Promise<void> {
const runner = context.deps.connection ?? (await import('../connection.js')).runKtxConnection;
context.setExitCode(await runner(args, context.io));
}
async function runMappingArgs(context: KtxCliCommandContext, args: KtxConnectionMappingArgs): Promise<void> {
const { runKtxConnectionMapping } = await import('./connection-mapping.js');
context.setExitCode(await runKtxConnectionMapping(args, context.io));
}
export function registerConnectionCommands(program: Command, context: KtxCliCommandContext, commandName = 'connection'): void {
const connection = program
.command(commandName)
.description('Add, list, test, and map data sources')
.description('List and test configured connections')
.showHelpAfterError()
.addHelpText(
'after',
@ -83,264 +41,4 @@ export function registerConnectionCommands(program: Command, context: KtxCliComm
connectionId,
});
});
connection
.command('add')
.description('Add or replace a configured connection')
.argument('<driver>', 'Connection driver')
.argument('<connectionId>', 'KTX connection id')
.option('--url <url>', 'Connection URL, env:NAME, or file:/path reference')
.option('--schema <schema>', 'Schema to include; repeatable', collectOption, [])
.option('--readonly', 'Mark the connection as read-only', false)
.option('--force', 'Replace an existing connection', false)
.option('--allow-literal-credentials', 'Allow writing a literal credential URL to ktx.yaml', false)
.addOption(new Option('--token-env <name>', 'Environment variable containing Notion auth token').conflicts('tokenFile'))
.addOption(new Option('--token-file <path>', 'File containing Notion auth token').conflicts('tokenEnv'))
.addOption(
new Option('--crawl-mode <mode>', 'Notion crawl mode: all_accessible or selected_roots')
.choices(CRAWL_MODE_CHOICES)
.default('selected_roots'),
)
.option('--root-page-id <id>', 'Root page to crawl; repeatable', collectOption, [])
.option('--root-database-id <id>', 'Root database to crawl; repeatable', collectOption, [])
.option('--root-data-source-id <id>', 'Root data source to crawl; repeatable', collectOption, [])
.option('--max-pages <n>', 'Maximum pages per run', parsePositiveIntegerOption)
.option('--max-knowledge-creates <n>', 'Maximum knowledge creates per run', parseNonNegativeIntegerOption)
.option('--max-knowledge-updates <n>', 'Maximum knowledge updates per run', parseNonNegativeIntegerOption)
.action(async (driver: string, connectionId: string, options, command) => {
const notion =
driver === 'notion'
? {
authTokenRef: options.tokenEnv
? `env:${options.tokenEnv}`
: options.tokenFile
? `file:${options.tokenFile}`
: '',
crawlMode: options.crawlMode,
rootPageIds: options.rootPageId,
rootDatabaseIds: options.rootDatabaseId,
rootDataSourceIds: options.rootDataSourceId,
maxPagesPerRun: options.maxPages,
maxKnowledgeCreatesPerRun: options.maxKnowledgeCreates,
maxKnowledgeUpdatesPerRun: options.maxKnowledgeUpdates,
}
: undefined;
if (driver === 'notion' && !notion?.authTokenRef) {
throw new Error('connection add notion requires --token-env NAME or --token-file PATH');
}
if (
driver === 'notion' &&
notion?.crawlMode === 'selected_roots' &&
notion.rootPageIds.length + notion.rootDatabaseIds.length + notion.rootDataSourceIds.length === 0
) {
throw new Error('connection add notion selected_roots requires at least one root id');
}
const args = connectionAddCommandSchema.parse({
command: 'add',
projectDir: resolveCommandProjectDir(command),
driver,
connectionId,
url: options.url,
schemas: options.schema.filter(Boolean),
readonly: options.readonly === true,
force: options.force === true,
allowLiteralCredentials: options.allowLiteralCredentials === true,
notion,
});
await runConnectionArgs(context, args);
});
connection
.command('remove')
.description('Remove a configured connection from ktx.yaml')
.argument('<connectionId>', 'KTX connection id')
.option('--force', 'Remove without prompting', false)
.option('--no-input', 'Disable interactive terminal input')
.action(async (connectionId: string, options: { force?: boolean; input?: boolean }, command) => {
await runConnectionArgs(context, {
command: 'remove',
projectDir: resolveCommandProjectDir(command),
connectionId,
force: options.force === true,
...(options.input === false ? { inputMode: 'disabled' } : {}),
});
});
connection
.command('map')
.description('Refresh and validate BI-to-warehouse mappings')
.argument('<sourceConnectionId>', 'Source BI connection id')
.option('--json', 'Print JSON output', false)
.action(async (sourceConnectionId: string, options: { json?: boolean }, command) => {
await runConnectionArgs(context, {
command: 'map',
projectDir: resolveCommandProjectDir(command),
sourceConnectionId,
json: options.json === true,
});
});
registerConnectionMappingCommands(connection, context);
registerConnectionMetabaseCommands(connection, context);
registerConnectionNotionCommands(connection, context);
}
export function registerConnectionMappingCommands(connection: Command, context: KtxCliCommandContext): void {
const mapping = connection
.command('mapping')
.description('Manage Metabase warehouse mappings')
.showHelpAfterError()
.addHelpText(
'after',
'\nProject directory defaults to KTX_PROJECT_DIR when set, otherwise the current working directory.\n',
);
mapping
.command('list')
.description('List Metabase database mappings')
.argument('<connectionId>', 'Metabase connection id')
.option('--json', 'Print JSON output where supported', false)
.action(async (connectionId: string, options: { json?: boolean }, command) => {
await runMappingArgs(context, {
command: 'list',
projectDir: resolveCommandProjectDir(command),
connectionId,
json: options.json === true,
});
});
mapping
.command('set')
.description('Set a Metabase or Looker warehouse mapping')
.argument('<connectionId>', 'Source connection id', parseSafeConnectionIdOption)
.argument('<field>', 'Mapping field', parseMappingFieldOption)
.argument('<assignment>', 'Mapping assignment such as 1=prod-warehouse', parseNonEmptyAssignmentOption)
.action(
async (
connectionId: string,
field: 'databaseMappings' | 'connectionMappings',
assignment: { key: string; value: string },
_options: unknown,
command,
) => {
await runMappingArgs(context, {
command: 'set',
projectDir: resolveCommandProjectDir(command),
connectionId,
field,
key: assignment.key,
value: assignment.value,
});
},
);
mapping
.command('apply-bulk')
.description('Apply mappings from JSON')
.argument('<connectionId>', 'Metabase connection id')
.requiredOption('--file <path>', 'JSON mapping file')
.action(async (connectionId: string, options: { file: string }, command) => {
await runMappingArgs(context, {
command: 'apply-bulk',
projectDir: resolveCommandProjectDir(command),
connectionId,
filePath: options.file,
});
});
mapping
.command('set-sync-enabled')
.description('Enable or disable sync for one Metabase database')
.argument('<connectionId>', 'Metabase connection id')
.argument('<metabaseDatabaseId>', 'Metabase database id', parsePositiveIntegerOption)
.requiredOption('--enabled <value>', 'true or false', parseBooleanStringOption)
.action(
async (connectionId: string, metabaseDatabaseId: number, options: { enabled: boolean }, command) => {
await runMappingArgs(context, {
command: 'set-sync-enabled',
projectDir: resolveCommandProjectDir(command),
connectionId,
metabaseDatabaseId,
enabled: options.enabled,
});
},
);
const syncState = mapping.command('sync-state').description('Manage Metabase sync-state selection');
syncState
.command('get')
.description('Read sync-state selection')
.argument('<connectionId>', 'Metabase connection id')
.option('--json', 'Print JSON output where supported', false)
.action(async (connectionId: string, options: { json?: boolean }, command) => {
await runMappingArgs(context, {
command: 'sync-state-get',
projectDir: resolveCommandProjectDir(command),
connectionId,
json: options.json === true,
});
});
syncState
.command('set')
.description('Write sync-state selection')
.argument('<connectionId>', 'Metabase connection id')
.addOption(new Option('--mode <mode>', 'ALL, ONLY, or EXCEPT').choices(SYNC_MODE_CHOICES).makeOptionMandatory())
.option('--collections <ids>', 'Comma-separated collection ids', parseCsvIds, [])
.option('--items <ids>', 'Comma-separated item ids', parseCsvIds, [])
.option('--tag-names <names>', 'Comma-separated tag names', parseCsvStrings, [])
.action(async (connectionId: string, options, command) => {
await runMappingArgs(context, {
command: 'sync-state-set',
projectDir: resolveCommandProjectDir(command),
connectionId,
syncMode: options.mode,
collectionIds: options.collections,
itemIds: options.items,
tagNames: options.tagNames,
});
});
mapping
.command('refresh')
.description('Refresh Metabase database mappings')
.argument('<connectionId>', 'Metabase connection id')
.option('--auto-accept', 'Accept refresh changes without prompting', false)
.action(async (connectionId: string, options: { autoAccept?: boolean }, command) => {
await runMappingArgs(context, {
command: 'refresh',
projectDir: resolveCommandProjectDir(command),
connectionId,
autoAccept: options.autoAccept === true,
});
});
mapping
.command('validate')
.description('Validate Metabase database mappings')
.argument('<connectionId>', 'Metabase connection id')
.action(async (connectionId: string, _options: unknown, command) => {
await runMappingArgs(context, {
command: 'validate',
projectDir: resolveCommandProjectDir(command),
connectionId,
});
});
mapping
.command('clear')
.description('Clear Metabase database mappings')
.argument('<connectionId>', 'Metabase connection id')
.argument('[metabaseDatabaseId]', 'Metabase database id', parsePositiveIntegerOption)
.action(async (connectionId: string, metabaseDatabaseId: number | undefined, _options: unknown, command) => {
await runMappingArgs(context, {
command: 'clear',
projectDir: resolveCommandProjectDir(command),
connectionId,
...(metabaseDatabaseId ? { metabaseDatabaseId } : {}),
});
});
}

329
packages/cli/src/commands/connection-mapping.test.ts
View file

@ -1,329 +0,0 @@
import { mkdtemp, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { LocalMetabaseSourceStateReader } from '@ktx/context/ingest';
import { initKtxProject, loadKtxProject, serializeKtxProjectConfig } from '@ktx/context/project';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { runKtxConnectionMapping } from './connection-mapping.js';
function makeIo() {
let stdout = '';
let stderr = '';
return {
io: {
stdout: {
write: (chunk: string) => {
stdout += chunk;
},
},
stderr: {
write: (chunk: string) => {
stderr += chunk;
},
},
},
stdout: () => stdout,
stderr: () => stderr,
};
}
describe('runKtxConnectionMapping', () => {
let tempDir: string;
let projectDir: string;
beforeEach(async () => {
tempDir = await mkdtemp(join(tmpdir(), 'ktx-cli-metabase-mapping-'));
projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'mapping' });
const project = await loadKtxProject({ projectDir });
await project.fileStore.writeFile(
'ktx.yaml',
serializeKtxProjectConfig({
...project.config,
connections: {
'prod-metabase': {
driver: 'metabase',
api_url: 'https://metabase.example.com',
api_key_ref: 'env:METABASE_API_KEY', // pragma: allowlist secret
},
'prod-warehouse': {
driver: 'postgres',
url: 'env:WAREHOUSE_URL',
readonly: true,
},
},
}),
'ktx',
'ktx@example.com',
'Seed Metabase mapping test connections',
);
});
async function replaceConnections(connections: Record<string, { driver: string; [key: string]: unknown }>) {
const project = await loadKtxProject({ projectDir });
await project.fileStore.writeFile(
'ktx.yaml',
serializeKtxProjectConfig({
...project.config,
connections,
}),
'ktx',
'ktx@example.com',
'Replace mapping test connections',
);
}
afterEach(async () => {
await rm(tempDir, { recursive: true, force: true });
});
it('sets, lists, disables, and clears local Metabase mappings', async () => {
const io = makeIo();
await expect(
runKtxConnectionMapping(
{
command: 'set',
projectDir,
connectionId: 'prod-metabase',
field: 'databaseMappings',
key: '1',
value: 'prod-warehouse',
},
io.io,
),
).resolves.toBe(0);
const listIo = makeIo();
await expect(
runKtxConnectionMapping({ command: 'list', projectDir, connectionId: 'prod-metabase', json: false }, listIo.io),
).resolves.toBe(0);
expect(listIo.stdout()).toContain('1 -> prod-warehouse');
expect(listIo.stdout()).toContain('unhydrated');
await expect(
runKtxConnectionMapping(
{
command: 'set-sync-enabled',
projectDir,
connectionId: 'prod-metabase',
metabaseDatabaseId: 1,
enabled: false,
},
makeIo().io,
),
).resolves.toBe(0);
await expect(
runKtxConnectionMapping(
{
command: 'clear',
projectDir,
connectionId: 'prod-metabase',
metabaseDatabaseId: 1,
},
makeIo().io,
),
).resolves.toBe(0);
});
it('lists Metabase yaml mapping bootstrap rows before any SQLite command writes', async () => {
const projectDir = await mkdtemp(join(tmpdir(), 'ktx-cli-yaml-mapping-'));
await initKtxProject({ projectDir, projectName: 'yaml-mapping' });
const project = await loadKtxProject({ projectDir });
await project.fileStore.writeFile(
'ktx.yaml',
serializeKtxProjectConfig({
...project.config,
connections: {
'prod-metabase': {
driver: 'metabase',
mappings: {
databaseMappings: { '1': 'prod-warehouse' },
syncEnabled: { '1': true },
},
},
'prod-warehouse': { driver: 'postgres', url: 'postgresql://readonly@db.test/analytics' },
},
}),
'ktx',
'ktx@example.com',
'Seed yaml mappings',
);
const io = makeIo();
await expect(
runKtxConnectionMapping(
{ command: 'list', projectDir, connectionId: 'prod-metabase', json: false },
io.io,
),
).resolves.toBe(0);
expect(io.stdout()).toContain('1 -> prod-warehouse');
expect(io.stdout()).toContain('source: ktx.yaml');
});
it('refreshes Metabase discovery metadata through the injected runtime client', async () => {
const client = {
getDatabases: vi.fn().mockResolvedValue([
{
id: 1,
name: 'Analytics',
engine: 'postgres',
details: { host: 'pg.internal', dbname: 'analytics' },
is_sample: false,
},
]),
cleanup: vi.fn(),
};
const io = makeIo();
await expect(
runKtxConnectionMapping(
{
command: 'refresh',
projectDir,
connectionId: 'prod-metabase',
autoAccept: true,
},
io.io,
{
createMetabaseClient: async () => client as never,
},
),
).resolves.toBe(0);
expect(io.stdout()).toContain('Discovery: 1 database');
expect(client.cleanup).toHaveBeenCalledTimes(1);
const store = new LocalMetabaseSourceStateReader({ dbPath: join(projectDir, '.ktx', 'db.sqlite') });
await expect(store.listDatabaseMappings('prod-metabase')).resolves.toMatchObject([
{ metabaseDatabaseId: 1, metabaseDatabaseName: 'Analytics', source: 'refresh' },
]);
});
it('sets and lists Looker connection mappings', async () => {
await replaceConnections({
'prod-looker': {
driver: 'looker',
base_url: 'https://looker.example.test',
client_id: 'id',
},
'prod-warehouse': {
driver: 'postgres',
url: 'postgresql://readonly@db.example.test/analytics',
},
});
const io = makeIo();
await expect(
runKtxConnectionMapping(
{
command: 'set',
projectDir,
connectionId: 'prod-looker',
field: 'connectionMappings',
key: 'analytics',
value: 'prod-warehouse',
},
io.io,
),
).resolves.toBe(0);
await expect(
runKtxConnectionMapping({ command: 'list', projectDir, connectionId: 'prod-looker', json: false }, io.io),
).resolves.toBe(0);
expect(io.stdout()).toContain('analytics -> prod-warehouse');
});
it('keeps driver-specific mapping field validation in the runner', async () => {
await replaceConnections({
'prod-looker': { driver: 'looker', base_url: 'https://looker.example.com' },
warehouse: { driver: 'postgres', url: 'env:WAREHOUSE_URL' },
});
const io = makeIo();
await expect(
runKtxConnectionMapping(
{
command: 'set',
projectDir,
connectionId: 'prod-looker',
field: 'databaseMappings',
key: '1',
value: 'warehouse',
},
io.io,
),
).resolves.toBe(1);
expect(io.stderr()).toContain('Looker mapping set requires connectionMappings');
});
it('refreshes Looker mapping metadata and reports drift', async () => {
await replaceConnections({
'prod-looker': {
driver: 'looker',
base_url: 'https://looker.example.test',
client_id: 'id',
},
'prod-warehouse': {
driver: 'postgres',
url: 'postgresql://readonly@db.example.test/analytics',
},
});
const io = makeIo();
await expect(
runKtxConnectionMapping(
{ command: 'refresh', projectDir, connectionId: 'prod-looker', autoAccept: true },
io.io,
{
createLookerClient: async () => ({
listLookerConnections: async () => [
{
name: 'analytics',
host: 'db.example.test',
database: 'analytics',
schema: null,
dialect: 'postgres',
},
],
cleanup: async () => {},
}),
},
),
).resolves.toBe(0);
expect(io.stdout()).toContain('Discovery: 1 connection');
expect(io.stdout()).toContain('Unmapped discovered: 1');
});
it('validates Looker mappings through the canonical local warehouse descriptor', async () => {
const projectDir = await mkdtemp(join(tmpdir(), 'ktx-cli-descriptor-validation-'));
await initKtxProject({ projectDir, projectName: 'descriptor-validation' });
const project = await loadKtxProject({ projectDir });
await project.fileStore.writeFile(
'ktx.yaml',
serializeKtxProjectConfig({
...project.config,
connections: {
'prod-looker': {
driver: 'looker',
mappings: { connectionMappings: { analytics: 'prod-warehouse' } },
},
'prod-warehouse': { driver: 'postgresql', url: 'postgresql://readonly@db.test/analytics' },
},
}),
'ktx',
'ktx@example.com',
'Seed descriptor validation',
);
const io = makeIo();
await expect(
runKtxConnectionMapping({ command: 'validate', projectDir, connectionId: 'prod-looker' }, io.io),
).resolves.toBe(0);
expect(io.stdout()).toContain('Mapping validation passed: prod-looker');
expect(io.stderr()).toBe('');
});
});

426
packages/cli/src/commands/connection-mapping.ts
View file

@ -1,426 +0,0 @@
import { readFile } from 'node:fs/promises';
import { localConnectionToWarehouseDescriptor } from '@ktx/context/connections';
import {
DEFAULT_METABASE_CLIENT_CONFIG,
DefaultLookerConnectionClientFactory,
DefaultMetabaseConnectionClientFactory,
LocalLookerRuntimeStore,
LocalMetabaseSourceStateReader,
computeLookerMappingDrift,
computeMetabaseMappingDrift,
discoverLookerConnections,
discoverMetabaseDatabases,
lookerCredentialsFromLocalConnection,
metabaseRuntimeConfigFromLocalConnection,
seedLocalMappingStateFromKtxYaml,
validateLookerMappings,
validateMappingPhysicalMatch,
type LookerMappingClient,
type MetabaseRuntimeClient,
type MetabaseSyncMode,
} from '@ktx/context/ingest';
import { type KtxLocalProject, ktxLocalStateDbPath, loadKtxProject } from '@ktx/context/project';
import type { KtxCliIo } from '../index.js';
import { profileMark } from '../startup-profile.js';
profileMark('module:commands/connection-mapping');
export type KtxConnectionMappingArgs =
| { command: 'list'; projectDir: string; connectionId: string; json: boolean }
| {
command: 'set';
projectDir: string;
connectionId: string;
field: 'databaseMappings' | 'connectionMappings';
key: string;
value: string;
}
| { command: 'apply-bulk'; projectDir: string; connectionId: string; filePath: string }
| {
command: 'set-sync-enabled';
projectDir: string;
connectionId: string;
metabaseDatabaseId: number;
enabled: boolean;
}
| { command: 'sync-state-get'; projectDir: string; connectionId: string; json: boolean }
| {
command: 'sync-state-set';
projectDir: string;
connectionId: string;
syncMode: MetabaseSyncMode;
collectionIds: number[];
itemIds: number[];
tagNames: string[];
}
| { command: 'refresh'; projectDir: string; connectionId: string; autoAccept: boolean }
| { command: 'validate'; projectDir: string; connectionId: string }
| { command: 'clear'; projectDir: string; connectionId: string; metabaseDatabaseId?: number; mappingKey?: string };
interface KtxConnectionMappingDeps {
createMetabaseClient?: (
project: KtxLocalProject,
connectionId: string,
) => Promise<Pick<MetabaseRuntimeClient, 'getDatabases' | 'cleanup'>>;
createLookerClient?: (
project: KtxLocalProject,
connectionId: string,
) => Promise<Pick<LookerMappingClient, 'listLookerConnections'> & { cleanup?(): Promise<void> }>;
}
interface MetabaseBulkMappingPayload {
databaseMappings?: Record<string, string | null>;
syncEnabled?: Record<string, boolean>;
syncMode?: MetabaseSyncMode;
selections?: { collections?: number[]; items?: number[] };
defaultTagNames?: string[];
}
function parseId(value: string, label: string): number {
const parsed = Number(value);
if (!Number.isInteger(parsed) || parsed < 1) {
throw new Error(`${label} must be a positive integer`);
}
return parsed;
}
async function createDefaultMetabaseClient(
project: KtxLocalProject,
connectionId: string,
): Promise<Pick<MetabaseRuntimeClient, 'getDatabases' | 'cleanup'>> {
const factory = new DefaultMetabaseConnectionClientFactory(
(metabaseConnectionId) =>
metabaseRuntimeConfigFromLocalConnection(metabaseConnectionId, project.config.connections[metabaseConnectionId]),
DEFAULT_METABASE_CLIENT_CONFIG,
);
return factory.createClient(connectionId);
}
async function createDefaultLookerClient(
project: KtxLocalProject,
connectionId: string,
): Promise<Pick<LookerMappingClient, 'listLookerConnections'> & { cleanup?(): Promise<void> }> {
const factory = new DefaultLookerConnectionClientFactory({
async resolve(lookerConnectionId) {
return lookerCredentialsFromLocalConnection(lookerConnectionId, project.config.connections[lookerConnectionId]);
},
});
return factory.createClient(connectionId) as unknown as Pick<LookerMappingClient, 'listLookerConnections'> & {
cleanup?(): Promise<void>;
};
}
function isLookerConnection(project: KtxLocalProject, connectionId: string): boolean {
return String(project.config.connections[connectionId]?.driver ?? '').toLowerCase() === 'looker';
}
function assertLookerConnection(project: KtxLocalProject, connectionId: string): void {
if (!isLookerConnection(project, connectionId)) {
throw new Error(`Connection "${connectionId}" is not a Looker connection`);
}
}
function assertMetabaseConnection(project: KtxLocalProject, connectionId: string): void {
const connection = project.config.connections[connectionId];
if (!connection || String(connection.driver).toLowerCase() !== 'metabase') {
throw new Error(`Connection "${connectionId}" is not a Metabase connection`);
}
}
function assertTargetConnection(project: KtxLocalProject, connectionId: string): void {
if (!project.config.connections[connectionId]) {
throw new Error(`Target connection "${connectionId}" does not exist`);
}
}
function targetPhysicalInfo(project: KtxLocalProject, connectionId: string) {
const descriptor = localConnectionToWarehouseDescriptor(connectionId, project.config.connections[connectionId]);
if (!descriptor) {
return { connection_type: 'UNKNOWN' };
}
return {
connection_type: descriptor.connection_type,
host: descriptor.host ?? null,
database: descriptor.database ?? null,
account: descriptor.account ?? null,
project_id: descriptor.project_id ?? null,
dataset_id: descriptor.dataset_id ?? null,
...descriptor.connection_params,
};
}
function renderMapping(
row: Awaited<ReturnType<LocalMetabaseSourceStateReader['listDatabaseMappings']>>[number],
): string {
const name = row.metabaseDatabaseName ?? 'unhydrated';
const target = row.targetConnectionId ?? '[unmapped]';
return `${row.metabaseDatabaseId} -> ${target} (${name}, sync: ${row.syncEnabled ? 'on' : 'off'}, source: ${
row.source
})`;
}
function renderLookerMapping(row: Awaited<ReturnType<LocalLookerRuntimeStore['listConnectionMappings']>>[number]): string {
const target = row.ktxConnectionId ?? '[unmapped]';
const metadata = [row.lookerDialect, row.lookerHost, row.lookerDatabase].filter(Boolean).join(', ');
return `${row.lookerConnectionName} -> ${target}${metadata ? ` (${metadata}, source: ${row.source})` : ` (source: ${row.source})`}`;
}
export async function runKtxConnectionMapping(
args: KtxConnectionMappingArgs,
io: KtxCliIo = process,
deps: KtxConnectionMappingDeps = {},
): Promise<number> {
try {
const project = await loadKtxProject({ projectDir: args.projectDir });
await seedLocalMappingStateFromKtxYaml(project, args.connectionId);
if (isLookerConnection(project, args.connectionId)) {
assertLookerConnection(project, args.connectionId);
const store = new LocalLookerRuntimeStore({ dbPath: ktxLocalStateDbPath(project) });
if (args.command === 'list') {
const rows = await store.listConnectionMappings(args.connectionId);
io.stdout.write(args.json ? `${JSON.stringify(rows, null, 2)}\n` : `${rows.map(renderLookerMapping).join('\n')}\n`);
return 0;
}
if (args.command === 'set') {
if (args.field !== 'connectionMappings') {
throw new Error('Looker mapping set requires connectionMappings <lookerConnectionName>=<targetConnectionId>');
}
assertTargetConnection(project, args.value);
await store.upsertConnectionMapping({
lookerConnectionId: args.connectionId,
lookerConnectionName: args.key,
ktxConnectionId: args.value,
source: 'cli',
});
io.stdout.write(`Set connectionMappings.${args.key} = ${args.value}\n`);
return 0;
}
if (args.command === 'refresh') {
const client = await (deps.createLookerClient ?? createDefaultLookerClient)(project, args.connectionId);
try {
const discovered = await discoverLookerConnections(client);
const drift = computeLookerMappingDrift({
storedMappings: await store.readMappings(args.connectionId),
discovered,
});
if (args.autoAccept) {
await store.refreshDiscoveredConnections({ lookerConnectionId: args.connectionId, discovered });
}
io.stdout.write(`Discovery: ${discovered.length} ${discovered.length === 1 ? 'connection' : 'connections'}\n`);
io.stdout.write(`Unmapped discovered: ${drift.unmappedDiscovered.length}\n`);
io.stdout.write(`Stale mappings: ${drift.staleMappings.length}\n`);
return 0;
} finally {
await client.cleanup?.();
}
}
if (args.command === 'validate') {
const knownKtxConnectionIds = new Set(Object.keys(project.config.connections));
const knownConnectionTypes = new Map(
Object.entries(project.config.connections).map(([id, _config]) => [id, targetPhysicalInfo(project, id).connection_type]),
);
const validation = validateLookerMappings({
mappings: await store.readMappings(args.connectionId),
knownKtxConnectionIds,
knownConnectionTypes,
});
if (!validation.ok) {
for (const error of validation.errors) {
io.stderr.write(`${error.key}: ${error.reason}\n`);
}
return 1;
}
io.stdout.write(`Mapping validation passed: ${args.connectionId}\n`);
return 0;
}
if (args.command === 'clear') {
await store.clearConnectionMappings({
lookerConnectionId: args.connectionId,
lookerConnectionName: args.mappingKey ?? (args.metabaseDatabaseId ? String(args.metabaseDatabaseId) : undefined),
});
io.stdout.write(
args.mappingKey
? `Cleared connectionMappings.${args.mappingKey}\n`
: `Cleared mappings for ${args.connectionId}\n`,
);
return 0;
}
throw new Error(`Looker connection mapping does not support ${args.command}`);
}
assertMetabaseConnection(project, args.connectionId);
const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(project) });
if (args.command === 'list') {
const rows = await store.listDatabaseMappings(args.connectionId);
io.stdout.write(args.json ? `${JSON.stringify(rows, null, 2)}\n` : `${rows.map(renderMapping).join('\n')}\n`);
return 0;
}
if (args.command === 'set') {
assertTargetConnection(project, args.value);
await store.upsertDatabaseMapping({
connectionId: args.connectionId,
metabaseDatabaseId: parseId(args.key, 'metabaseDatabaseId'),
targetConnectionId: args.value,
syncEnabled: true,
source: 'cli',
});
io.stdout.write(`Set databaseMappings.${args.key} = ${args.value}\n`);
return 0;
}
if (args.command === 'apply-bulk') {
const payload = JSON.parse(await readFile(args.filePath, 'utf8')) as MetabaseBulkMappingPayload;
const existingState = await store.getSourceState(args.connectionId);
const existingRows = await store.listDatabaseMappings(args.connectionId);
const existingById = new Map(existingRows.map((row) => [row.metabaseDatabaseId, row]));
const databaseMappings = payload.databaseMappings ?? {};
for (const targetConnectionId of Object.values(databaseMappings)) {
if (targetConnectionId) {
assertTargetConnection(project, targetConnectionId);
}
}
const mappingIds = new Set([
...existingRows.map((row) => row.metabaseDatabaseId),
...Object.keys(databaseMappings).map((id) => parseId(id, 'metabaseDatabaseId')),
...Object.keys(payload.syncEnabled ?? {}).map((id) => parseId(id, 'metabaseDatabaseId')),
]);
await store.replaceSourceState({
connectionId: args.connectionId,
syncMode: payload.syncMode ?? existingState.syncMode,
defaultTagNames: payload.defaultTagNames ?? existingState.defaultTagNames,
selections:
payload.selections === undefined
? existingState.selections
: [
...(payload.selections.collections ?? []).map((id) => ({
selectionType: 'collection' as const,
metabaseObjectId: id,
})),
...(payload.selections.items ?? []).map((id) => ({
selectionType: 'item' as const,
metabaseObjectId: id,
})),
],
mappings: [...mappingIds]
.sort((a, b) => a - b)
.map((id) => {
const existing = existingById.get(id);
return {
metabaseDatabaseId: id,
metabaseDatabaseName: existing?.metabaseDatabaseName ?? null,
metabaseEngine: existing?.metabaseEngine ?? null,
metabaseHost: existing?.metabaseHost ?? null,
metabaseDbName: existing?.metabaseDbName ?? null,
targetConnectionId: databaseMappings[String(id)] ?? existing?.targetConnectionId ?? null,
syncEnabled: payload.syncEnabled?.[String(id)] ?? existing?.syncEnabled ?? false,
source: 'cli',
};
}),
});
io.stdout.write(`Applied bulk mappings for ${args.connectionId}\n`);
return 0;
}
if (args.command === 'set-sync-enabled') {
await store.setMappingSyncEnabled({
connectionId: args.connectionId,
metabaseDatabaseId: args.metabaseDatabaseId,
syncEnabled: args.enabled,
});
io.stdout.write(`Set syncEnabled.${args.metabaseDatabaseId} = ${args.enabled}\n`);
return 0;
}
if (args.command === 'sync-state-get') {
const state = await store.getSourceState(args.connectionId);
const payload = {
syncMode: state.syncMode,
selections: state.selections,
defaultTagNames: state.defaultTagNames,
};
io.stdout.write(args.json ? `${JSON.stringify(payload, null, 2)}\n` : `${payload.syncMode}\n`);
return 0;
}
if (args.command === 'sync-state-set') {
await store.setSyncState({
connectionId: args.connectionId,
syncMode: args.syncMode,
defaultTagNames: args.tagNames,
selections: [
...args.collectionIds.map((id) => ({ selectionType: 'collection' as const, metabaseObjectId: id })),
...args.itemIds.map((id) => ({ selectionType: 'item' as const, metabaseObjectId: id })),
],
});
io.stdout.write(`Set sync state for ${args.connectionId}\n`);
return 0;
}
if (args.command === 'refresh') {
const client = await (deps.createMetabaseClient ?? createDefaultMetabaseClient)(project, args.connectionId);
try {
const discovered = await discoverMetabaseDatabases(client);
const existing = Object.fromEntries(
(await store.listDatabaseMappings(args.connectionId)).map((row) => [
String(row.metabaseDatabaseId),
row.targetConnectionId,
]),
);
const drift = computeMetabaseMappingDrift({ currentMappings: existing, discovered });
if (args.autoAccept) {
await store.refreshDiscoveredDatabases({ connectionId: args.connectionId, discovered });
}
io.stdout.write(`Discovery: ${discovered.length} ${discovered.length === 1 ? 'database' : 'databases'}\n`);
io.stdout.write(`Unmapped discovered: ${drift.unmappedDiscovered.length}\n`);
io.stdout.write(`Stale mappings: ${drift.staleMappings.length}\n`);
return 0;
} finally {
await client.cleanup();
}
}
if (args.command === 'validate') {
const rows = await store.listDatabaseMappings(args.connectionId);
const failures = rows.flatMap((row) => {
if (!row.targetConnectionId) {
return [];
}
const reason = validateMappingPhysicalMatch(
{ metabaseEngine: row.metabaseEngine, metabaseDbName: row.metabaseDbName, metabaseHost: row.metabaseHost },
project.config.connections[row.targetConnectionId]
? targetPhysicalInfo(project, row.targetConnectionId)
: { connection_type: 'UNKNOWN' },
);
return reason ? [`${row.metabaseDatabaseId}: ${reason}`] : [];
});
if (failures.length > 0) {
for (const failure of failures) {
io.stderr.write(`${failure}\n`);
}
return 1;
}
io.stdout.write(`Mapping validation passed: ${args.connectionId}\n`);
return 0;
}
const metabaseDatabaseId = args.metabaseDatabaseId ?? (args.mappingKey ? parseId(args.mappingKey, 'metabaseDatabaseId') : undefined);
await store.clearDatabaseMappings({ connectionId: args.connectionId, metabaseDatabaseId });
io.stdout.write(
metabaseDatabaseId
? `Cleared databaseMappings.${metabaseDatabaseId}\n`
: `Cleared mappings for ${args.connectionId}\n`,
);
return 0;
} catch (error) {
io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
return 1;
}
}

132
packages/cli/src/commands/connection-metabase-commands.ts
View file

@ -1,132 +0,0 @@
import { type Command, Option } from '@commander-js/extra-typings';
import {
type KtxCliCommandContext,
parseNonEmptyAssignmentOption,
parsePositiveIntegerOption,
parseSafeConnectionIdOption,
resolveCommandProjectDir,
} from '../cli-program.js';
import {
type KtxConnectionMetabaseSetupArgs,
type MetabaseSetupMappingAssignment,
type MetabaseSetupSyncMode,
runKtxConnectionMetabaseSetup,
} from './connection-metabase-setup.js';
const SYNC_MODE_CHOICES = ['ALL', 'ONLY', 'EXCEPT'] as const satisfies readonly MetabaseSetupSyncMode[];
interface ConnectionMetabaseSetupOptions {
id?: string;
url?: string;
apiKey?: string;
mintApiKey?: boolean;
username?: string;
password?: string;
map: MetabaseSetupMappingAssignment[];
sync: number[];
syncMode: MetabaseSetupSyncMode;
runIngest?: boolean;
yes?: boolean;
input?: boolean;
}
function collectPositiveIntegerOption(value: string, previous: number[] = []): number[] {
return [...previous, parsePositiveIntegerOption(value)];
}
function parseMappingAssignment(value: string): MetabaseSetupMappingAssignment {
const assignment = parseNonEmptyAssignmentOption(value);
return {
metabaseDatabaseId: parsePositiveIntegerOption(assignment.key),
targetConnectionId: parseSafeConnectionIdOption(assignment.value),
};
}
function collectMappingOption(
value: string,
previous: MetabaseSetupMappingAssignment[] = [],
): MetabaseSetupMappingAssignment[] {
return [...previous, parseMappingAssignment(value)];
}
async function runMetabaseSetupArgs(
context: KtxCliCommandContext,
args: KtxConnectionMetabaseSetupArgs,
): Promise<void> {
const runner = context.deps.connectionMetabaseSetup ?? runKtxConnectionMetabaseSetup;
context.setExitCode(await runner(args, context.io));
}
export function registerConnectionMetabaseCommands(connection: Command, context: KtxCliCommandContext): void {
const metabase = connection
.command('metabase')
.description('Configure Metabase connections')
.showHelpAfterError()
.addHelpText(
'after',
'\nProject directory defaults to KTX_PROJECT_DIR when set, otherwise the current working directory.\n',
);
metabase.action(() => {
metabase.outputHelp();
context.setExitCode(0);
});
metabase
.command('setup')
.description('Guided setup for a Metabase connection')
.option('--id <connectionId>', 'KTX connection id to write', parseSafeConnectionIdOption)
.option('--url <url>', 'Metabase API URL')
.addOption(new Option('--api-key <key>', 'Metabase API key').conflicts('mintApiKey'))
.option('--mint-api-key', 'Mint a Metabase API key with credentials', false)
.option('--username <email>', 'Metabase admin username for API-key minting')
.option('--password <password>', 'Metabase admin password for API-key minting')
.addHelpText(
'after',
'\nGuided equivalent of:\n' +
' ktx connection mapping refresh <connectionId> --auto-accept\n' +
' ktx connection mapping set <connectionId> databaseMappings <id>=<target>\n' +
' ktx connection mapping set-sync-enabled <connectionId> <id> --enabled true\n' +
' ktx ingest <connectionId>\n',
)
.option(
'--map <metabaseDatabaseId=targetConnectionId>',
'Assign a Metabase database id to a warehouse connection; repeatable',
collectMappingOption,
[],
)
.option(
'--sync <metabaseDatabaseId>',
'Enable Metabase sync for a discovered database; repeatable',
collectPositiveIntegerOption,
[],
)
.addOption(
new Option('--sync-mode <mode>', 'Metabase sync selection mode')
.choices(SYNC_MODE_CHOICES)
.default('ALL' satisfies MetabaseSetupSyncMode),
)
.option('--run-ingest', 'Run ingest after setup', false)
.option('--yes', 'Confirm and apply setup changes without prompting', false)
.option('--no-input', 'Disable interactive terminal input')
.showHelpAfterError()
.action(async (options: ConnectionMetabaseSetupOptions, command) => {
await runMetabaseSetupArgs(context, {
command: 'setup',
projectDir: resolveCommandProjectDir(command),
connectionId: options.id,
url: options.url,
apiKey: options.apiKey,
mintApiKey: options.mintApiKey === true,
metabaseUsername: options.username,
metabasePassword: options.password,
mappings: options.map,
syncEnabledDatabaseIds: options.sync,
syncMode: options.syncMode ?? 'ALL',
runIngest: options.runIngest === true,
yes: options.yes === true,
inputMode: options.input === false ? 'disabled' : 'auto',
});
});
}

1136
packages/cli/src/commands/connection-metabase-setup.test.ts
View file

File diff suppressed because it is too large Load diff

782
packages/cli/src/commands/connection-metabase-setup.ts
View file

@ -1,782 +0,0 @@
import type { Option as ClackOption } from '@clack/prompts';
import {
cancel,
confirm,
intro,
isCancel,
log,
multiselect,
note,
outro,
password,
select,
text,
} from '@clack/prompts';
import { localConnectionToWarehouseDescriptor } from '@ktx/context/connections';
import {
DEFAULT_METABASE_CLIENT_CONFIG,
DefaultMetabaseConnectionClientFactory,
LocalMetabaseSourceStateReader,
MetabaseClient,
type MetabaseDatabase,
type MetabaseRuntimeClient,
type MetabaseSyncMode,
metabaseRuntimeConfigFromLocalConnection,
validateMappingPhysicalMatch,
} from '@ktx/context/ingest';
import {
type KtxLocalProject,
type KtxProjectConnectionConfig,
ktxLocalStateDbPath,
loadKtxProject,
serializeKtxProjectConfig,
} from '@ktx/context/project';
import { createClackSpinner, type KtxCliSpinner } from '../clack.js';
import type { KtxCliIo } from '../cli-runtime.js';
import { withMenuOptionsSpacing, withMultiselectNavigation } from '../prompt-navigation.js';
import { type KtxPublicIngestArgs, runKtxPublicIngest } from '../public-ingest.js';
export type KtxMetabaseSetupInputMode = 'auto' | 'disabled';
export type MetabaseSetupSyncMode = MetabaseSyncMode;
type MetabaseSetupPromptOption<Value> = ClackOption<Value>;
export interface MetabaseSetupLogger {
info(message: string): void;
step(message: string): void;
success(message: string): void;
warn(message: string): void;
error(message: string): void;
}
export interface MetabaseSetupPromptAdapter {
intro(title?: string): void;
outro(message?: string): void;
note(message: string, title: string): void;
log: MetabaseSetupLogger;
spinner(): KtxCliSpinner;
select<T extends string>(options: { message: string; options: Array<MetabaseSetupPromptOption<T>> }): Promise<T>;
multiselect<Value extends number | string>(options: {
message: string;
options: Array<MetabaseSetupPromptOption<Value>>;
initialValues?: Value[];
required?: boolean;
maxItems?: number;
}): Promise<Value[]>;
text(options: { message: string; placeholder?: string }): Promise<string>;
password(options: { message: string }): Promise<string>;
confirm(options: { message: string; initialValue?: boolean }): Promise<boolean>;
cancel(message: string): void;
}
type KtxMetabaseSetupInteractiveIo = KtxCliIo & {
stdin?: { isTTY?: boolean };
};
export interface MetabaseSetupMappingAssignment {
metabaseDatabaseId: number;
targetConnectionId: string;
}
export interface MintMetabaseApiKeyArgs {
url: string;
username: string;
password: string;
}
export type MintMetabaseApiKey = (args: MintMetabaseApiKeyArgs, io: KtxCliIo) => Promise<string>;
export interface KtxConnectionMetabaseSetupArgs {
command: 'setup';
projectDir: string;
connectionId?: string;
url?: string;
apiKey?: string;
mintApiKey: boolean;
metabaseUsername?: string;
metabasePassword?: string;
mappings: MetabaseSetupMappingAssignment[];
syncEnabledDatabaseIds: number[];
syncMode: MetabaseSetupSyncMode;
runIngest: boolean;
yes: boolean;
inputMode: KtxMetabaseSetupInputMode;
}
export interface KtxConnectionMetabaseSetupDeps {
createMetabaseClient?: (
project: KtxLocalProject,
connectionId: string,
) => Promise<Pick<MetabaseRuntimeClient, 'testConnection' | 'getDatabases' | 'cleanup'>>;
mintMetabaseApiKey?: MintMetabaseApiKey;
prompts?: MetabaseSetupPromptAdapter;
runPublicIngest?: (args: Extract<KtxPublicIngestArgs, { command: 'run' }>, io: KtxCliIo) => Promise<number>;
}
function isMetabaseConnection(connection: KtxProjectConnectionConfig | undefined): boolean {
return (
String(connection?.driver ?? '')
.trim()
.toLowerCase() === 'metabase'
);
}
function stringField(value: unknown): string | undefined {
return typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined;
}
function uniqueSorted(values: number[]): number[] {
return [...new Set(values)].sort((a, b) => a - b);
}
function resolveMetabaseUrl(connection: KtxProjectConnectionConfig | undefined): string | undefined {
return stringField(connection?.api_url) ?? stringField(connection?.apiUrl) ?? stringField(connection?.url);
}
function resolveLiteralMetabaseApiKey(connection: KtxProjectConnectionConfig | undefined): string | undefined {
return stringField(connection?.api_key) ?? stringField(connection?.apiKey);
}
function listMetabaseConnectionIds(project: KtxLocalProject): string[] {
return Object.entries(project.config.connections)
.filter(([_connectionId, connection]) => isMetabaseConnection(connection))
.map(([connectionId]) => connectionId)
.sort();
}
function listWarehouseConnectionIds(project: KtxLocalProject): string[] {
return Object.entries(project.config.connections)
.filter(([connectionId, connection]) => localConnectionToWarehouseDescriptor(connectionId, connection) != null)
.map(([connectionId]) => connectionId)
.sort();
}
function redactSecrets(message: string, secrets: string[]): string {
let result = message;
for (const secret of secrets) {
if (!secret) {
continue;
}
result = result.split(secret).join('[redacted]');
}
return result;
}
async function createDefaultMetabaseClient(
project: KtxLocalProject,
connectionId: string,
): Promise<Pick<MetabaseRuntimeClient, 'testConnection' | 'getDatabases' | 'cleanup'>> {
const factory = new DefaultMetabaseConnectionClientFactory(
(metabaseConnectionId) =>
metabaseRuntimeConfigFromLocalConnection(metabaseConnectionId, project.config.connections[metabaseConnectionId]),
DEFAULT_METABASE_CLIENT_CONFIG,
);
return factory.createClient(connectionId);
}
async function defaultMintMetabaseApiKey(args: MintMetabaseApiKeyArgs): Promise<string> {
const loginClient = new MetabaseClient({ apiUrl: args.url, apiKey: '' }, DEFAULT_METABASE_CLIENT_CONFIG);
const sessionId = await loginClient.createSession(args.username, args.password);
const sessionClient = new MetabaseClient(
{ apiUrl: args.url, apiKey: sessionId, authHeaderName: 'X-Metabase-Session' },
DEFAULT_METABASE_CLIENT_CONFIG,
);
const groups = await sessionClient.getPermissionGroups();
const adminGroup = groups.find((group) => group.name === 'Administrators');
if (!adminGroup) {
throw new Error('Metabase Administrators group was not found; create an API key manually and pass --api-key');
}
const mintedKey = await sessionClient.createApiKey({
groupId: adminGroup.id,
name: `KTX CLI ${new Date().toISOString()}`,
});
const trimmedKey = stringField(mintedKey);
if (!trimmedKey) {
throw new Error('Metabase API key minting returned an empty key');
}
return trimmedKey;
}
function ensureNotCancelled<T>(value: T | symbol, prompts: Pick<MetabaseSetupPromptAdapter, 'cancel'>): T {
if (isCancel(value)) {
prompts.cancel('Setup cancelled.');
throw new Error('Setup cancelled.');
}
return value as T;
}
export function createClackMetabaseSetupPromptAdapter(): MetabaseSetupPromptAdapter {
return {
intro(title?: string): void {
intro(title);
},
outro(message?: string): void {
outro(message);
},
note(message: string, title: string): void {
note(message, title);
},
log: {
info(message: string): void {
log.info(message);
},
step(message: string): void {
log.step(message);
},
success(message: string): void {
log.success(message);
},
warn(message: string): void {
log.warn(message);
},
error(message: string): void {
log.error(message);
},
},
spinner(): KtxCliSpinner {
return createClackSpinner();
},
async select<T extends string>(options: {
message: string;
options: Array<MetabaseSetupPromptOption<T>>;
}): Promise<T> {
return ensureNotCancelled(await select(withMenuOptionsSpacing(options)), this);
},
async multiselect<Value extends number | string>(options: {
message: string;
options: Array<MetabaseSetupPromptOption<Value>>;
initialValues?: Value[];
required?: boolean;
maxItems?: number;
}): Promise<Value[]> {
return ensureNotCancelled(await multiselect(withMenuOptionsSpacing(options)), this);
},
async text(options: { message: string; placeholder?: string }): Promise<string> {
return ensureNotCancelled(await text(options), this);
},
async password(options: { message: string }): Promise<string> {
return ensureNotCancelled(await password(options), this);
},
async confirm(options: { message: string; initialValue?: boolean }): Promise<boolean> {
return ensureNotCancelled(await confirm(options), this);
},
cancel(message: string): void {
cancel(message);
},
};
}
function isInteractiveMetabaseSetupIo(
args: Pick<KtxConnectionMetabaseSetupArgs, 'inputMode'>,
io: KtxMetabaseSetupInteractiveIo,
): boolean {
return args.inputMode !== 'disabled' && io.stdin?.isTTY === true && io.stdout.isTTY === true;
}
function normalizeDiscoveredDatabases(databases: MetabaseDatabase[]): Array<{
id: number;
name: string;
engine: string;
host: string | null;
dbName: string | null;
}> {
return databases
.filter((database) => database.is_sample !== true)
.map((database) => ({
id: database.id,
name: database.name,
engine: stringField(database.engine) ?? 'unknown',
host: stringField(database.details?.host) ?? null,
dbName: stringField(database.details?.dbname) ?? null,
}));
}
function targetPhysicalInfo(project: KtxLocalProject, connectionId: string) {
const descriptor = localConnectionToWarehouseDescriptor(connectionId, project.config.connections[connectionId]);
if (!descriptor) {
return { connection_type: 'UNKNOWN' };
}
return {
connection_type: descriptor.connection_type,
host: descriptor.host ?? null,
database: descriptor.database ?? null,
account: descriptor.account ?? null,
project_id: descriptor.project_id ?? null,
dataset_id: descriptor.dataset_id ?? null,
...descriptor.connection_params,
};
}
function noteMetabaseSetupSummary(options: {
prompts: MetabaseSetupPromptAdapter;
connectionId: string;
url: string;
mappings: MetabaseSetupMappingAssignment[];
syncEnabledDatabaseIds: number[];
}): void {
const mappingLines = options.mappings
.map((mapping) => ` ${mapping.metabaseDatabaseId} -> ${mapping.targetConnectionId}`)
.join('\n');
const syncLines = options.syncEnabledDatabaseIds.map((id) => ` ${id}`).join('\n');
options.prompts.note(
[
`Connection: ${options.connectionId}`,
`URL: ${options.url}`,
'',
'Mappings:',
mappingLines || ' (none)',
'',
'Sync enabled:',
syncLines || ' (none)',
].join('\n'),
'Summary',
);
}
export async function runKtxConnectionMetabaseSetup(
args: KtxConnectionMetabaseSetupArgs,
io: KtxCliIo,
deps: KtxConnectionMetabaseSetupDeps = {},
): Promise<number> {
let apiKeyForRedaction = args.apiKey;
let passwordForRedaction = args.metabasePassword;
const interactiveIo = io as KtxMetabaseSetupInteractiveIo;
const isInteractive = isInteractiveMetabaseSetupIo(args, interactiveIo);
const prompts = deps.prompts ?? (isInteractive ? createClackMetabaseSetupPromptAdapter() : undefined);
try {
if (isInteractive && prompts) {
prompts.intro('KTX Metabase setup');
}
const project = await loadKtxProject({ projectDir: args.projectDir });
const existingMetabaseConnectionIds = listMetabaseConnectionIds(project);
let connectionId: string;
if (args.connectionId) {
connectionId = args.connectionId;
} else if (existingMetabaseConnectionIds.length === 1) {
const onlyMetabaseConnectionId = existingMetabaseConnectionIds[0];
if (!onlyMetabaseConnectionId) {
throw new Error('No Metabase connection id was resolved');
}
connectionId = onlyMetabaseConnectionId;
} else if (existingMetabaseConnectionIds.length > 1) {
if (!isInteractive || !prompts) {
throw new Error(
`Multiple Metabase connections found (${existingMetabaseConnectionIds.join(', ')}); select one with --id`,
);
}
connectionId = await prompts.select({
message: 'Select the Metabase connection to configure',
options: existingMetabaseConnectionIds.map((id) => ({ value: id, label: id })),
});
} else {
connectionId = 'metabase';
}
const existingConnection = project.config.connections[connectionId];
const warehouseConnectionIds = listWarehouseConnectionIds(project);
if (warehouseConnectionIds.length === 0) {
throw new Error('Add a warehouse connection first');
}
let url = args.url ?? resolveMetabaseUrl(existingConnection);
let apiKey = args.apiKey ?? resolveLiteralMetabaseApiKey(existingConnection);
apiKeyForRedaction = apiKey;
if (!url && isInteractive && prompts) {
url = stringField(
await prompts.text({
message: 'Metabase API URL',
placeholder: 'http://localhost:3000',
}),
);
}
if (args.inputMode === 'disabled' && !url) {
throw new Error('missing Metabase URL');
}
if (!args.apiKey && !args.mintApiKey && apiKey && isInteractive && prompts && !args.yes) {
const reuse = await prompts.confirm({
message: `Reuse the existing Metabase API key from connections.${connectionId}?`,
initialValue: true,
});
if (!reuse) {
apiKey = undefined;
apiKeyForRedaction = undefined;
}
}
if (args.mintApiKey) {
let username = stringField(args.metabaseUsername);
let metabasePassword = stringField(args.metabasePassword);
if (isInteractive && prompts) {
if (!username) {
username = stringField(await prompts.text({ message: 'Metabase admin username' }));
}
if (!metabasePassword) {
metabasePassword = stringField(await prompts.password({ message: 'Metabase admin password' }));
}
}
if (!username) {
throw new Error('--mint-api-key requires --username');
}
if (!metabasePassword) {
throw new Error('--mint-api-key requires --password');
}
if (!url) {
throw new Error('Metabase URL is required (use --url)');
}
passwordForRedaction = metabasePassword;
apiKey = await (deps.mintMetabaseApiKey ?? defaultMintMetabaseApiKey)(
{ url, username, password: metabasePassword },
io,
);
apiKeyForRedaction = apiKey;
}
if (!apiKey && isInteractive && prompts) {
const credentialMode = await prompts.select({
message: 'Metabase credentials',
options: [
{ value: 'paste', label: 'Paste API key' },
{ value: 'mint', label: 'Mint API key' },
],
});
if (credentialMode === 'paste') {
apiKey = stringField(await prompts.password({ message: 'Metabase API key' }));
apiKeyForRedaction = apiKey;
} else {
const username = stringField(await prompts.text({ message: 'Metabase admin username' }));
const metabasePassword = stringField(await prompts.password({ message: 'Metabase admin password' }));
if (!username) {
throw new Error('Metabase username is required');
}
if (!metabasePassword) {
throw new Error('Metabase password is required');
}
if (!url) {
throw new Error('Metabase URL is required (use --url)');
}
passwordForRedaction = metabasePassword;
apiKey = await (deps.mintMetabaseApiKey ?? defaultMintMetabaseApiKey)(
{ url, username, password: metabasePassword },
io,
);
apiKeyForRedaction = apiKey;
}
}
if (args.inputMode === 'disabled' && !apiKey) {
throw new Error('missing Metabase API key');
}
if (!url) {
throw new Error('Metabase URL is required (use --url)');
}
if (!apiKey) {
throw new Error('Metabase API key is required (use --api-key)');
}
const transientConnectionConfig: KtxProjectConnectionConfig = {
...(existingConnection ?? {}),
driver: 'metabase',
api_url: url,
api_key: apiKey,
};
const configWithTransient = {
...project.config,
connections: {
...project.config.connections,
[connectionId]: transientConnectionConfig,
},
};
const discoveryProject: KtxLocalProject = { ...project, config: configWithTransient };
for (const mapping of args.mappings) {
if (!configWithTransient.connections[mapping.targetConnectionId]) {
throw new Error(`Target connection "${mapping.targetConnectionId}" does not exist`);
}
}
const client = await (deps.createMetabaseClient ?? createDefaultMetabaseClient)(discoveryProject, connectionId);
try {
const authSpinner = isInteractive && prompts ? prompts.spinner() : undefined;
authSpinner?.start('Testing Metabase connection');
const testResult = await client.testConnection();
if (!testResult.success) {
authSpinner?.error('Metabase authentication failed');
throw new Error(
`Metabase authentication failed. Replace connections.${connectionId}.api_key or use --mint-api-key.`,
);
}
authSpinner?.stop('Metabase reachable');
const discoverySpinner = isInteractive && prompts ? prompts.spinner() : undefined;
discoverySpinner?.start('Discovering Metabase databases');
const discovered = normalizeDiscoveredDatabases(await client.getDatabases());
discoverySpinner?.stop(`Discovered ${discovered.length} ${discovered.length === 1 ? 'database' : 'databases'}`);
if (isInteractive && prompts) {
prompts.log.success(
`Discovered ${discovered.length} ${discovered.length === 1 ? 'database' : 'databases'}`,
);
}
if (discovered.length === 0) {
throw new Error('Metabase auth worked but no usable databases were returned');
}
let resolvedMappings = args.mappings;
let resolvedSyncEnabledDatabaseIds = args.syncEnabledDatabaseIds;
if (resolvedSyncEnabledDatabaseIds.length === 0 && args.yes && resolvedMappings.length > 0) {
resolvedSyncEnabledDatabaseIds = uniqueSorted(resolvedMappings.map((mapping) => mapping.metabaseDatabaseId));
}
if (resolvedMappings.length === 0 && resolvedSyncEnabledDatabaseIds.length === 0) {
const onlyDiscoveredDatabase = discovered.length === 1 ? discovered[0] : undefined;
const compatibleWarehouses = onlyDiscoveredDatabase
? warehouseConnectionIds.filter((warehouseConnectionId) => {
const mismatchReason = validateMappingPhysicalMatch(
{
metabaseEngine: onlyDiscoveredDatabase.engine,
metabaseDbName: onlyDiscoveredDatabase.dbName,
metabaseHost: onlyDiscoveredDatabase.host,
},
targetPhysicalInfo(project, warehouseConnectionId),
);
return !mismatchReason;
})
: [];
const onlyWarehouseConnectionId = compatibleWarehouses[0];
if (onlyDiscoveredDatabase && compatibleWarehouses.length === 1 && onlyWarehouseConnectionId) {
if (args.yes) {
resolvedMappings = [
{ metabaseDatabaseId: onlyDiscoveredDatabase.id, targetConnectionId: onlyWarehouseConnectionId },
];
resolvedSyncEnabledDatabaseIds = [onlyDiscoveredDatabase.id];
} else if (isInteractive && prompts) {
const proposedMappings = [
{ metabaseDatabaseId: onlyDiscoveredDatabase.id, targetConnectionId: onlyWarehouseConnectionId },
];
const proposedSyncEnabledDatabaseIds = [onlyDiscoveredDatabase.id];
noteMetabaseSetupSummary({
prompts,
connectionId,
url,
mappings: proposedMappings,
syncEnabledDatabaseIds: proposedSyncEnabledDatabaseIds,
});
const confirmed = await prompts.confirm({
message: `Map Metabase database "${onlyDiscoveredDatabase.name}" (${onlyDiscoveredDatabase.id}) to "${onlyWarehouseConnectionId}" and enable sync?`,
initialValue: true,
});
if (!confirmed) {
prompts.cancel('Setup cancelled.');
throw new Error('Setup cancelled.');
}
resolvedMappings = proposedMappings;
resolvedSyncEnabledDatabaseIds = proposedSyncEnabledDatabaseIds;
} else {
throw new Error('Metabase mapping/sync is required in --no-input mode; pass --map and --sync');
}
} else if (isInteractive && prompts) {
const selectedDatabaseIds = await prompts.multiselect<number>({
message: withMultiselectNavigation('Select Metabase databases to configure'),
options: discovered.map((database) => ({
value: database.id,
label: `${database.id}: ${database.name}`,
hint: [database.engine, database.host, database.dbName].filter(Boolean).join(' • '),
})),
required: true,
});
resolvedMappings = [];
for (const databaseId of selectedDatabaseIds) {
const database = discovered.find((candidate) => candidate.id === databaseId);
if (!database) {
throw new Error(`Selected database id ${databaseId} was not discovered`);
}
const existingMapping = args.mappings.find((mapping) => mapping.metabaseDatabaseId === databaseId);
if (existingMapping) {
resolvedMappings.push(existingMapping);
continue;
}
const targetConnectionId = await prompts.select({
message: `Map Metabase database ${database.id} ("${database.name}") to which KTX connection?`,
options: warehouseConnectionIds.map((warehouseId) => ({ value: warehouseId, label: warehouseId })),
});
resolvedMappings.push({ metabaseDatabaseId: databaseId, targetConnectionId });
}
const syncIds = await prompts.multiselect<number>({
message: withMultiselectNavigation('Enable sync for which databases?'),
options: selectedDatabaseIds.map((id) => ({ value: id, label: String(id) })),
initialValues: selectedDatabaseIds,
required: true,
});
resolvedSyncEnabledDatabaseIds = uniqueSorted(syncIds);
if (!args.yes) {
noteMetabaseSetupSummary({
prompts,
connectionId,
url,
mappings: resolvedMappings,
syncEnabledDatabaseIds: resolvedSyncEnabledDatabaseIds,
});
const confirmed = await prompts.confirm({
message: 'Write changes to ktx.yaml and enable sync?',
initialValue: true,
});
if (!confirmed) {
prompts.cancel('Setup cancelled.');
throw new Error('Setup cancelled.');
}
}
} else if (args.inputMode === 'disabled') {
throw new Error('Metabase mapping/sync is required in --no-input mode; pass --map and --sync');
}
}
if (
args.inputMode === 'disabled' &&
resolvedMappings.length > 0 &&
resolvedSyncEnabledDatabaseIds.length === 0
) {
throw new Error('Metabase sync selection is required in --no-input mode; pass --sync <metabaseDatabaseId>');
}
const discoveredIds = new Set(discovered.map((database) => database.id));
for (const mapping of resolvedMappings) {
if (!discoveredIds.has(mapping.metabaseDatabaseId)) {
throw new Error(`Mapped database id ${mapping.metabaseDatabaseId} was not discovered`);
}
}
for (const syncId of resolvedSyncEnabledDatabaseIds) {
if (!discoveredIds.has(syncId)) {
throw new Error(`Sync database id ${syncId} was not discovered`);
}
}
await project.fileStore.writeFile(
'ktx.yaml',
serializeKtxProjectConfig(configWithTransient),
'ktx',
'ktx@example.com',
`Setup Metabase connection ${connectionId}`,
);
const updatedProject = await loadKtxProject({ projectDir: args.projectDir });
const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(updatedProject) });
await store.refreshDiscoveredDatabases({ connectionId, discovered });
for (const mapping of resolvedMappings) {
await store.upsertDatabaseMapping({
connectionId,
metabaseDatabaseId: mapping.metabaseDatabaseId,
targetConnectionId: mapping.targetConnectionId,
syncEnabled: false,
source: 'cli',
});
}
for (const metabaseDatabaseId of resolvedSyncEnabledDatabaseIds) {
await store.setMappingSyncEnabled({
connectionId,
metabaseDatabaseId,
syncEnabled: true,
});
}
const existingSyncState = await store.getSourceState(connectionId);
await store.setSyncState({
connectionId,
syncMode: args.syncMode,
defaultTagNames: existingSyncState.defaultTagNames,
selections: existingSyncState.selections,
});
const unhydrated = await store.getUnhydratedSyncEnabledMappingIds(connectionId);
if (unhydrated.length > 0) {
io.stderr.write(
`Sync-enabled mappings are missing discovery metadata; run ktx connection mapping refresh ${connectionId} --auto-accept\n`,
);
return 1;
}
const rows = await store.listDatabaseMappings(connectionId);
const physicalFailures = rows.flatMap((row) => {
if (!row.targetConnectionId) {
return [];
}
const reason = validateMappingPhysicalMatch(
{ metabaseEngine: row.metabaseEngine, metabaseDbName: row.metabaseDbName, metabaseHost: row.metabaseHost },
updatedProject.config.connections[row.targetConnectionId]
? targetPhysicalInfo(updatedProject, row.targetConnectionId)
: { connection_type: 'UNKNOWN' },
);
return reason ? [`${row.metabaseDatabaseId}: ${reason}`] : [];
});
if (physicalFailures.length > 0) {
for (const failure of physicalFailures) {
io.stderr.write(`${failure}\n`);
}
return 1;
}
io.stdout.write(`Connection: ${connectionId}\n`);
io.stdout.write(`Discovered ${discovered.length} ${discovered.length === 1 ? 'database' : 'databases'}\n`);
io.stdout.write(`Next: ktx ingest ${connectionId} --project-dir ${args.projectDir}\n`);
if (args.runIngest) {
const ingestRunner = deps.runPublicIngest ?? runKtxPublicIngest;
const exitCode = await ingestRunner(
{
command: 'run',
projectDir: args.projectDir,
targetConnectionId: connectionId,
all: false,
json: false,
inputMode: 'disabled',
},
io,
);
if (exitCode !== 0) {
io.stderr.write(`Ingest failed; re-run: ktx ingest ${connectionId} --project-dir ${args.projectDir}\n`);
return 1;
}
}
if (isInteractive && prompts) {
prompts.outro('Metabase setup complete');
}
return 0;
} finally {
await client.cleanup();
}
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
io.stderr.write(
`${redactSecrets(message, [apiKeyForRedaction ?? '', passwordForRedaction ?? '', args.apiKey ?? ''])}\n`,
);
return 1;
}
}

92
packages/cli/src/commands/connection-notion-commands.ts
View file

@ -1,92 +0,0 @@
import { type Command, InvalidArgumentError } from '@commander-js/extra-typings';
import { collectOption, type KtxCliCommandContext, resolveCommandProjectDir } from '../cli-program.js';
import type { KtxConnectionNotionArgs } from './connection-notion.js';
interface NotionPickOptions {
input?: boolean;
rootPageId: string[];
}
function parseSafeConnectionId(value: string): string {
if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(value)) {
throw new InvalidArgumentError(`Unsafe connection id: ${value}`);
}
return value;
}
function uniqueInOrder(values: string[]): string[] {
const seen = new Set<string>();
const result: string[] = [];
for (const value of values) {
if (!seen.has(value)) {
seen.add(value);
result.push(value);
}
}
return result;
}
function normalizeNotionPageId(value: string): string {
const trimmed = value.trim();
const compact = trimmed.includes('-') ? trimmed.replace(/-/g, '') : trimmed;
if (!/^[0-9a-fA-F]{32}$/.test(compact)) {
throw new Error(`Invalid Notion page UUID: ${value}`);
}
const lower = compact.toLowerCase();
return `${lower.slice(0, 8)}-${lower.slice(8, 12)}-${lower.slice(12, 16)}-${lower.slice(16, 20)}-${lower.slice(20)}`;
}
function buildPickArgs(connectionId: string, projectDir: string, options: NotionPickOptions): KtxConnectionNotionArgs {
if (options.input !== false) {
return {
command: 'pick',
projectDir,
connectionId,
mode: 'interactive',
};
}
const rootPageIds = uniqueInOrder(options.rootPageId.map(normalizeNotionPageId));
if (rootPageIds.length === 0) {
throw new Error('connection notion pick --no-input requires at least one --root-page-id');
}
return {
command: 'pick',
projectDir,
connectionId,
mode: 'non-interactive',
rootPageIds,
};
}
async function runConnectionNotionArgs(context: KtxCliCommandContext, args: KtxConnectionNotionArgs): Promise<void> {
const runner = context.deps.connectionNotion ?? (await import('./connection-notion.js')).runKtxConnectionNotion;
context.setExitCode(await runner(args, context.io));
}
export function registerConnectionNotionCommands(connect: Command, context: KtxCliCommandContext): void {
const notion = connect
.command('notion')
.description('Configure Notion source selection')
.showHelpAfterError()
.addHelpText(
'after',
'\nProject directory defaults to KTX_PROJECT_DIR when set, otherwise the current working directory.\n',
);
notion.action(() => {
notion.outputHelp();
context.setExitCode(0);
});
notion
.command('pick')
.description('Pick Notion root pages for a configured Notion connection')
.argument('<connectionId>', 'Notion connection id', parseSafeConnectionId)
.option('--no-input', 'Disable interactive terminal input')
.option('--root-page-id <id>', 'Root page UUID to crawl; repeatable with --no-input', collectOption, [])
.showHelpAfterError()
.action(async (connectionId: string, options: NotionPickOptions, command) => {
await runConnectionNotionArgs(context, buildPickArgs(connectionId, resolveCommandProjectDir(command), options));
});
}

513
packages/cli/src/commands/connection-notion.test.ts
View file

@ -1,513 +0,0 @@
import { mkdtemp, readFile, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import {
initKtxProject,
loadKtxProject,
serializeKtxProjectConfig,
type KtxProjectConfig,
} from '@ktx/context/project';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import {
applyNotionPickerWriteback,
discoverNotionPickerPages,
notionPickerPageFromSearchResult,
normalizeNotionPageId,
resolveNotionWorkspaceLabel,
runKtxConnectionNotion,
type NotionPickerApi,
type PickerRenderInput,
type PickerRenderResult,
} from './connection-notion.js';
function makeIo() {
let stdout = '';
let stderr = '';
return {
io: {
stdout: {
write: (chunk: string) => {
stdout += chunk;
},
},
stderr: {
write: (chunk: string) => {
stderr += chunk;
},
},
},
stdout: () => stdout,
stderr: () => stderr,
};
}
type FakeNotionSearchPage = Record<string, unknown> & { id: string; object: 'page' };
const PAGE_IDS = {
engineering: '11111111-1111-1111-1111-111111111111',
architecture: '22222222-2222-2222-2222-222222222222',
stale: '99999999-9999-9999-9999-999999999999',
};
function notionPage(id: string, title: string, parentId: string | null = null): FakeNotionSearchPage {
return {
object: 'page',
id,
archived: false,
parent: parentId ? { type: 'page_id', page_id: parentId } : { type: 'workspace', workspace: true },
properties: {
title: {
type: 'title',
title: [{ plain_text: title }],
},
},
};
}
function fakeNotionApi(pages: FakeNotionSearchPage[]): NotionPickerApi {
return {
search: vi.fn(async (_filterValue, startCursor) => {
if (startCursor === 'page-2') {
return { results: pages.slice(2), hasMore: false, nextCursor: null };
}
return {
results: pages.slice(0, 2),
hasMore: pages.length > 2,
nextCursor: pages.length > 2 ? 'page-2' : null,
};
}),
retrieveBotUser: vi.fn(async () => ({ name: 'Notion bot', bot: { workspace_name: 'Design Workspace' } })),
};
}
describe('normalizeNotionPageId', () => {
it('accepts dashed and compact UUIDs', () => {
expect(normalizeNotionPageId('11111111222233334444555555555555')).toBe(
'11111111-2222-3333-4444-555555555555',
);
expect(normalizeNotionPageId('AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE')).toBe(
'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee',
);
});
});
describe('runKtxConnectionNotion', () => {
let tempDir: string;
beforeEach(async () => {
tempDir = await mkdtemp(join(tmpdir(), 'ktx-cli-notion-pick-'));
});
afterEach(async () => {
await rm(tempDir, { recursive: true, force: true });
});
async function writeProjectConfig(projectDir: string, config: KtxProjectConfig): Promise<void> {
const project = await loadKtxProject({ projectDir });
await project.fileStore.writeFile(
'ktx.yaml',
serializeKtxProjectConfig(config),
'ktx',
'ktx@example.com',
'seed test config',
);
}
it('rejects unsafe connection ids before loading a project', async () => {
const io = makeIo();
const loadProject = vi.fn(async () => {
throw new Error('loadProject should not be called');
});
await expect(
runKtxConnectionNotion(
{
command: 'pick',
projectDir: '/tmp/project',
connectionId: '../evil',
mode: 'interactive',
},
io.io,
{ loadProject },
),
).resolves.toBe(1);
expect(loadProject).not.toHaveBeenCalled();
expect(io.stderr()).toContain('Unsafe connection id: ../evil');
});
it('writes selected root_page_ids while preserving every other Notion connection field', async () => {
const projectDir = join(tempDir, 'project');
const initialized = await initKtxProject({ projectDir, projectName: 'warehouse' });
await writeProjectConfig(projectDir, {
...initialized.config,
connections: {
'notion-main': {
driver: 'notion',
auth_token_ref: 'env:NOTION_TOKEN',
crawl_mode: 'all_accessible',
root_page_ids: ['99999999-9999-9999-9999-999999999999'],
root_database_ids: ['database-1'],
root_data_source_ids: ['data-source-1'],
max_pages_per_run: 12,
max_knowledge_creates_per_run: 2,
max_knowledge_updates_per_run: 7,
last_successful_cursor: '{"phase":"all_accessible_pages","cursor":"cursor-1"}',
unknown_future_field: 'keep-me',
},
},
});
const io = makeIo();
await expect(
runKtxConnectionNotion(
{
command: 'pick',
projectDir,
connectionId: 'notion-main',
mode: 'non-interactive',
rootPageIds: [
'11111111-2222-3333-4444-555555555555',
'66666666-7777-8888-9999-aaaaaaaaaaaa',
],
},
io.io,
),
).resolves.toBe(0);
const yaml = await readFile(join(projectDir, 'ktx.yaml'), 'utf-8');
expect(yaml).toContain('crawl_mode: selected_roots');
expect(yaml).toContain('root_page_ids:');
expect(yaml).toContain('11111111-2222-3333-4444-555555555555');
expect(yaml).toContain('66666666-7777-8888-9999-aaaaaaaaaaaa');
expect(yaml).toContain('root_database_ids:');
expect(yaml).toContain('database-1');
expect(yaml).toContain('root_data_source_ids:');
expect(yaml).toContain('data-source-1');
expect(yaml).toContain('last_successful_cursor: \'{"phase":"all_accessible_pages","cursor":"cursor-1"}\'');
expect(yaml).toContain('unknown_future_field: keep-me');
expect(io.stdout()).toContain('Connection: notion-main');
expect(io.stdout()).toContain('rootPageIds: 2');
expect(io.stdout()).toContain('crawlMode: selected_roots');
});
it('rejects empty writeback, missing connections, and non-Notion connections', async () => {
const projectDir = join(tempDir, 'project');
const initialized = await initKtxProject({ projectDir, projectName: 'warehouse' });
await writeProjectConfig(projectDir, {
...initialized.config,
connections: {
warehouse: {
driver: 'postgres',
url: 'env:DATABASE_URL',
readonly: true,
},
},
});
const project = await loadKtxProject({ projectDir });
await expect(applyNotionPickerWriteback(project, 'warehouse', [])).rejects.toThrow(
'connection notion pick requires at least one root page id',
);
await expect(
applyNotionPickerWriteback(project, 'missing', ['11111111-2222-3333-4444-555555555555']),
).rejects.toThrow('Connection "missing" not found');
await expect(
applyNotionPickerWriteback(project, 'warehouse', ['11111111-2222-3333-4444-555555555555']),
).rejects.toThrow('Connection "warehouse" is not a Notion connection');
});
it('extracts picker page inputs from Notion search results', () => {
expect(notionPickerPageFromSearchResult(notionPage(PAGE_IDS.architecture, 'Architecture', PAGE_IDS.engineering)))
.toEqual({
id: PAGE_IDS.architecture,
title: 'Architecture',
archived: false,
parentId: PAGE_IDS.engineering,
});
expect(
notionPickerPageFromSearchResult({
object: 'page',
id: PAGE_IDS.engineering.replaceAll('-', ''),
archived: true,
parent: { type: 'workspace', workspace: true },
properties: {},
}),
).toEqual({
id: PAGE_IDS.engineering,
title: 'Untitled',
archived: true,
parentId: null,
});
});
it('discovers visible pages up to the cap and reports cap state', async () => {
const api = fakeNotionApi([
notionPage(PAGE_IDS.engineering, 'Engineering'),
notionPage(PAGE_IDS.architecture, 'Architecture', PAGE_IDS.engineering),
notionPage('33333333-3333-3333-3333-333333333333', 'Onboarding', PAGE_IDS.engineering),
]);
await expect(discoverNotionPickerPages(api, { cap: 2 })).resolves.toEqual({
pages: [
{ id: PAGE_IDS.engineering, title: 'Engineering', archived: false, parentId: null },
{ id: PAGE_IDS.architecture, title: 'Architecture', archived: false, parentId: PAGE_IDS.engineering },
],
cappedAtCount: 2,
warnings: [],
});
expect(api.search).toHaveBeenCalledTimes(1);
});
it('keeps partial discovery results when Notion search fails after at least one page', async () => {
const api: NotionPickerApi = {
search: vi
.fn()
.mockResolvedValueOnce({
results: [notionPage(PAGE_IDS.engineering, 'Engineering')],
hasMore: true,
nextCursor: 'cursor-2',
})
.mockRejectedValueOnce(new Error('rate limit after first page')),
retrieveBotUser: vi.fn(async () => ({ name: 'Notion bot' })),
};
await expect(discoverNotionPickerPages(api)).resolves.toEqual({
pages: [{ id: PAGE_IDS.engineering, title: 'Engineering', archived: false, parentId: null }],
cappedAtCount: null,
warnings: ['Notion search stopped early: rate limit after first page'],
});
});
it('uses the Notion workspace name when available and falls back to the connection id', async () => {
await expect(resolveNotionWorkspaceLabel(fakeNotionApi([]), 'notion-main')).resolves.toBe('Design Workspace');
await expect(
resolveNotionWorkspaceLabel(
{
search: vi.fn(),
retrieveBotUser: vi.fn(async () => {
throw new Error('users.me unavailable');
}),
},
'notion-main',
),
).resolves.toBe('notion-main');
});
it('runs interactive discovery, warns about stale roots, renders the TUI, and saves selected roots', async () => {
const projectDir = join(tempDir, 'project');
const initialized = await initKtxProject({ projectDir, projectName: 'warehouse' });
await writeProjectConfig(projectDir, {
...initialized.config,
connections: {
'notion-main': {
driver: 'notion',
auth_token_ref: 'env:NOTION_TOKEN',
crawl_mode: 'all_accessible',
root_page_ids: [PAGE_IDS.stale],
root_database_ids: ['database-1'],
root_data_source_ids: ['data-source-1'],
max_pages_per_run: 12,
max_knowledge_creates_per_run: 2,
max_knowledge_updates_per_run: 7,
last_successful_cursor: null,
},
},
});
const api = fakeNotionApi([
notionPage(PAGE_IDS.engineering, 'Engineering'),
notionPage(PAGE_IDS.architecture, 'Architecture', PAGE_IDS.engineering),
]);
const renderPicker = vi.fn(async (input): Promise<PickerRenderResult> => {
expect(input.connectionId).toBe('notion-main');
expect(input.workspaceLabel).toBe('Design Workspace');
expect(input.currentCrawlMode).toBe('all_accessible');
expect(input.cappedAtCount).toBeNull();
expect(input.initialState.preLoadWarnings).toEqual(['1 stored root_page_ids no longer visible']);
return { kind: 'save', rootPageIds: [PAGE_IDS.engineering] };
});
const io = makeIo();
await expect(
runKtxConnectionNotion(
{
command: 'pick',
projectDir,
connectionId: 'notion-main',
mode: 'interactive',
},
io.io,
{
env: { NOTION_TOKEN: 'ntn_test_token' },
createNotionApi: vi.fn(() => api),
renderPicker,
},
),
).resolves.toBe(0);
const yaml = await readFile(join(projectDir, 'ktx.yaml'), 'utf-8');
expect(yaml).toContain('crawl_mode: selected_roots');
expect(yaml).toContain(PAGE_IDS.engineering);
expect(yaml).not.toContain(PAGE_IDS.stale);
expect(io.stderr()).toContain('1 stored root_page_ids no longer visible');
expect(io.stdout()).toContain('Connection: notion-main');
expect(io.stdout()).toContain('rootPageIds: 1');
});
it('uses inline Notion auth_token for interactive discovery', async () => {
const projectDir = join(tempDir, 'project');
const initialized = await initKtxProject({ projectDir, projectName: 'warehouse' });
await writeProjectConfig(projectDir, {
...initialized.config,
connections: {
'notion-main': {
driver: 'notion',
auth_token: 'ntn_inline_token',
crawl_mode: 'selected_roots',
root_page_ids: [PAGE_IDS.engineering],
root_database_ids: [],
root_data_source_ids: [],
max_pages_per_run: 12,
max_knowledge_creates_per_run: 2,
max_knowledge_updates_per_run: 7,
last_successful_cursor: null,
},
},
});
const api = fakeNotionApi([notionPage(PAGE_IDS.engineering, 'Engineering')]);
const createNotionApi = vi.fn((authToken: string) => {
expect(authToken).toBe('ntn_inline_token');
return api;
});
const io = makeIo();
await expect(
runKtxConnectionNotion(
{
command: 'pick',
projectDir,
connectionId: 'notion-main',
mode: 'interactive',
},
io.io,
{
createNotionApi,
renderPicker: vi.fn(async (): Promise<PickerRenderResult> => ({ kind: 'quit' })),
},
),
).resolves.toBe(0);
expect(createNotionApi).toHaveBeenCalledOnce();
expect(io.stdout()).toContain('No changes saved.');
});
it('passes partial-discovery warnings into the TUI banner state', async () => {
const projectDir = join(tempDir, 'project');
const initialized = await initKtxProject({ projectDir, projectName: 'warehouse' });
await writeProjectConfig(projectDir, {
...initialized.config,
connections: {
'notion-main': {
driver: 'notion',
auth_token_ref: 'env:NOTION_TOKEN',
crawl_mode: 'selected_roots',
root_page_ids: [PAGE_IDS.engineering],
root_database_ids: [],
root_data_source_ids: [],
max_pages_per_run: 12,
max_knowledge_creates_per_run: 2,
max_knowledge_updates_per_run: 7,
last_successful_cursor: null,
},
},
});
const api: NotionPickerApi = {
search: vi
.fn()
.mockResolvedValueOnce({
results: [notionPage(PAGE_IDS.engineering, 'Engineering')],
hasMore: true,
nextCursor: 'cursor-2',
})
.mockRejectedValueOnce(new Error('rate limit after first page')),
retrieveBotUser: vi.fn(async () => ({ name: 'Notion bot', bot: { workspace_name: 'Design Workspace' } })),
};
let renderInput: PickerRenderInput | undefined;
const renderPicker = vi.fn(async (input: PickerRenderInput): Promise<PickerRenderResult> => {
renderInput = input;
return { kind: 'quit' };
});
const io = makeIo();
await expect(
runKtxConnectionNotion(
{
command: 'pick',
projectDir,
connectionId: 'notion-main',
mode: 'interactive',
},
io.io,
{
env: { NOTION_TOKEN: 'ntn_test_token' },
createNotionApi: vi.fn(() => api),
renderPicker,
},
),
).resolves.toBe(0);
expect(renderPicker).toHaveBeenCalledOnce();
if (!renderInput) {
throw new Error('renderPicker was not called');
}
expect(renderInput.initialState.preLoadWarnings).toEqual(['Notion search stopped early: rate limit after first page']);
expect(renderInput.initialState.tree.map((node) => node.title)).toEqual(['Engineering']);
expect(io.stderr()).toContain('Notion search stopped early: rate limit after first page');
expect(io.stdout()).toContain('No changes saved.');
});
it('quits interactive mode without writing when the TUI returns quit', async () => {
const projectDir = join(tempDir, 'project');
const initialized = await initKtxProject({ projectDir, projectName: 'warehouse' });
await writeProjectConfig(projectDir, {
...initialized.config,
connections: {
'notion-main': {
driver: 'notion',
auth_token_ref: 'env:NOTION_TOKEN',
crawl_mode: 'selected_roots',
root_page_ids: [PAGE_IDS.engineering],
root_database_ids: [],
root_data_source_ids: [],
max_pages_per_run: 12,
max_knowledge_creates_per_run: 2,
max_knowledge_updates_per_run: 7,
last_successful_cursor: null,
},
},
});
const before = await readFile(join(projectDir, 'ktx.yaml'), 'utf-8');
const io = makeIo();
await expect(
runKtxConnectionNotion(
{
command: 'pick',
projectDir,
connectionId: 'notion-main',
mode: 'interactive',
},
io.io,
{
env: { NOTION_TOKEN: 'ntn_test_token' },
createNotionApi: vi.fn(() => fakeNotionApi([notionPage(PAGE_IDS.engineering, 'Engineering')])),
renderPicker: vi.fn(async (): Promise<PickerRenderResult> => ({ kind: 'quit' })),
},
),
).resolves.toBe(0);
await expect(readFile(join(projectDir, 'ktx.yaml'), 'utf-8')).resolves.toBe(before);
expect(io.stdout()).toContain('No changes saved.');
});
});

53
packages/cli/src/commands/doctor-commands.ts
View file

@ -1,53 +0,0 @@
import type { Command } from '@commander-js/extra-typings';
import { type CommandWithGlobalOptions, type KtxCliCommandContext, resolveCommandProjectDir } from '../cli-program.js';
import type { KtxDoctorArgs } from '../doctor.js';
import { profileMark } from '../startup-profile.js';
profileMark('module:commands/doctor-commands');
function outputMode(options: { json?: boolean }): 'plain' | 'json' {
return options.json === true ? 'json' : 'plain';
}
function inputMode(options: { input?: boolean }): { inputMode?: 'disabled' } {
return options.input === false ? { inputMode: 'disabled' } : {};
}
async function runDoctorArgs(context: KtxCliCommandContext, args: KtxDoctorArgs): Promise<void> {
const runner = context.deps.doctor ?? (await import('../doctor.js')).runKtxDoctor;
context.setExitCode(await runner(args, context.io));
}
export function registerDoctorCommands(program: Command, context: KtxCliCommandContext): void {
const doctor = program
.command('doctor')
.description('Check KTX setup and project readiness')
.option('--json', 'Print JSON output', false)
.option('--no-input', 'Disable interactive terminal input')
.action(async (options: { json?: boolean; input?: boolean }, command) => {
await runDoctorArgs(context, {
command: 'project',
projectDir: resolveCommandProjectDir(command),
outputMode: outputMode(options),
...inputMode(options),
});
});
doctor
.command('setup')
.description('Check KTX install, build, and local runtime readiness')
.option('--json', 'Print JSON output', false)
.option('--no-input', 'Disable interactive terminal input')
.action(
async (
_options: { json?: boolean; input?: boolean },
command: CommandWithGlobalOptions,
) => {
const options = (command.optsWithGlobals ? command.optsWithGlobals() : command.opts()) as {
json?: boolean;
input?: boolean;
};
await runDoctorArgs(context, { command: 'setup', outputMode: outputMode(options), ...inputMode(options) });
},
);
}

21
packages/cli/src/commands/knowledge-commands.ts
View file

@ -1,5 +1,10 @@
import { type Command, Option } from '@commander-js/extra-typings';
import { collectOption, type KtxCliCommandContext, resolveCommandProjectDir } from '../cli-program.js';
import {
collectOption,
type KtxCliCommandContext,
parsePositiveIntegerOption,
resolveCommandProjectDir,
} from '../cli-program.js';
import { wikiWriteCommandSchema } from '../command-schemas.js';
import type { KtxKnowledgeArgs } from '../knowledge.js';
import { profileMark } from '../startup-profile.js';
@ -24,12 +29,14 @@ export function registerWikiCommands(program: Command, context: KtxCliCommandCon
wiki
.command('list')
.description('List local wiki pages')
.option('--json', 'Print JSON output', false)
.option('--user-id <id>', 'Local user id', 'local')
.action(async (options: { userId: string }, command) => {
.action(async (options: { userId: string; json?: boolean }, command) => {
await runKnowledgeArgs(context, {
command: 'list',
projectDir: resolveCommandProjectDir(command),
userId: options.userId,
json: options.json,
});
});
@ -37,13 +44,15 @@ export function registerWikiCommands(program: Command, context: KtxCliCommandCon
.command('read')
.description('Read one local wiki page')
.argument('<key>', 'Wiki page key')
.option('--json', 'Print JSON output', false)
.option('--user-id <id>', 'Local user id', 'local')
.action(async (key: string, options: { userId: string }, command) => {
.action(async (key: string, options: { userId: string; json?: boolean }, command) => {
await runKnowledgeArgs(context, {
command: 'read',
projectDir: resolveCommandProjectDir(command),
key,
userId: options.userId,
json: options.json,
});
});
@ -51,13 +60,17 @@ export function registerWikiCommands(program: Command, context: KtxCliCommandCon
.command('search')
.description('Search local wiki pages')
.argument('<query>', 'Search query')
.option('--json', 'Print JSON output', false)
.option('--user-id <id>', 'Local user id', 'local')
.action(async (query: string, options: { userId: string }, command) => {
.option('--limit <number>', 'Maximum search results', parsePositiveIntegerOption)
.action(async (query: string, options: { userId: string; json?: boolean; limit?: number }, command) => {
await runKnowledgeArgs(context, {
command: 'search',
projectDir: resolveCommandProjectDir(command),
query,
userId: options.userId,
json: options.json,
...(options.limit !== undefined ? { limit: options.limit } : {}),
});
});

109
packages/cli/src/commands/public-ingest-commands.ts
View file

@ -1,109 +0,0 @@
import { InvalidArgumentError, type Command } from '@commander-js/extra-typings';
import { type KtxCliCommandContext, resolveCommandProjectDir } from '../cli-program.js';
import { publicIngestReadCommandSchema, publicIngestRunCommandSchema } from '../command-schemas.js';
import type { KtxPublicIngestArgs, KtxPublicIngestInputMode } from '../public-ingest.js';
import { profileMark } from '../startup-profile.js';
profileMark('module:commands/public-ingest-commands');
interface PublicIngestOptions {
all?: boolean;
json?: boolean;
input?: boolean;
}
function inputMode(options: { input?: boolean }): KtxPublicIngestInputMode {
return options.input === false ? 'disabled' : 'auto';
}
async function runPublicIngestArgs(context: KtxCliCommandContext, args: KtxPublicIngestArgs): Promise<void> {
const runner = context.deps.publicIngest ?? (await import('../public-ingest.js')).runKtxPublicIngest;
context.setExitCode(await runner(args, context.io));
}
function parsePublicIngestConnectionId(value: string): string {
if (value === 'run') {
throw new InvalidArgumentError('run is reserved; use ktx dev ingest run for low-level adapter syntax');
}
return value;
}
export function registerPublicIngestCommands(program: Command, context: KtxCliCommandContext): void {
const ingest = program
.command('ingest')
.description('Build and refresh KTX context from configured sources')
.usage('[options] [connectionId]')
.argument('[connectionId]', 'Connection id to ingest', parsePublicIngestConnectionId)
.option('--all', 'Ingest every eligible configured source', false)
.option('--json', 'Print JSON output', false)
.option('--no-input', 'Disable interactive terminal input')
.addHelpText(
'after',
[
'',
'Examples:',
' ktx ingest <connectionId> [options]',
' ktx ingest --all [options]',
' ktx ingest status [runId] [options]',
' ktx ingest watch [runId] [options]',
'',
'Project directory defaults to KTX_PROJECT_DIR when set, otherwise the current working directory.',
'',
].join('\n'),
)
.showHelpAfterError()
.hook('preAction', (_thisCommand, actionCommand) => {
context.writeDebug?.('ingest', actionCommand);
})
.action(async (connectionId: string | undefined, _options: PublicIngestOptions, command) => {
const options = command.opts();
if (options.all === true && connectionId) {
throw new Error('ktx ingest accepts either --all or <connectionId>, not both');
}
const args = publicIngestRunCommandSchema.parse({
command: 'run',
projectDir: resolveCommandProjectDir(command),
...(connectionId ? { targetConnectionId: connectionId } : {}),
all: options.all === true,
json: options.json === true,
inputMode: inputMode(options),
});
await runPublicIngestArgs(context, args);
});
ingest
.command('status')
.description('Print status for the latest or selected public ingest run')
.argument('[runId]', 'Public ingest run id')
.option('--json', 'Print JSON output', false)
.option('--no-input', 'Disable interactive terminal input')
.action(async (runId: string | undefined, _options: PublicIngestOptions, command) => {
const options = (command.optsWithGlobals ? command.optsWithGlobals() : command.opts()) as PublicIngestOptions;
const args = publicIngestReadCommandSchema.parse({
command: 'status',
projectDir: resolveCommandProjectDir(command),
...(runId ? { runId } : {}),
json: options.json === true,
inputMode: inputMode(options),
});
await runPublicIngestArgs(context, args);
});
ingest
.command('watch')
.description('Open the latest or selected public ingest visual report')
.argument('[runId]', 'Public ingest run id')
.option('--json', 'Print JSON output instead of the visual report', false)
.option('--no-input', 'Disable interactive terminal input')
.action(async (runId: string | undefined, _options: PublicIngestOptions, command) => {
const options = (command.optsWithGlobals ? command.optsWithGlobals() : command.opts()) as PublicIngestOptions;
const args = publicIngestReadCommandSchema.parse({
command: 'watch',
projectDir: resolveCommandProjectDir(command),
...(runId ? { runId } : {}),
json: options.json === true,
inputMode: inputMode(options),
});
await runPublicIngestArgs(context, args);
});
}

30
packages/cli/src/commands/runtime-commands.ts
View file

@ -18,7 +18,7 @@ async function runRuntimeArgs(context: KtxCliCommandContext, args: KtxRuntimeArg
export function registerRuntimeCommands(program: Command, context: KtxCliCommandContext): void {
const runtime = program
.command('runtime')
.description('Install, inspect, and prune the KTX-managed Python runtime')
.description('Install, start, stop, and inspect the KTX-managed Python runtime')
.showHelpAfterError();
runtime
@ -64,7 +64,7 @@ export function registerRuntimeCommands(program: Command, context: KtxCliCommand
runtime
.command('status')
.description('Show managed Python runtime status')
.description('Show managed Python runtime status and readiness checks')
.option('--json', 'Print JSON output', false)
.action(async (options: { json?: boolean }) => {
await runRuntimeArgs(context, {
@ -73,30 +73,4 @@ export function registerRuntimeCommands(program: Command, context: KtxCliCommand
json: options.json === true,
});
});
runtime
.command('doctor')
.description('Check managed Python runtime prerequisites and installation')
.option('--json', 'Print JSON output', false)
.action(async (options: { json?: boolean }) => {
await runRuntimeArgs(context, {
command: 'doctor',
cliVersion: context.packageInfo.version,
json: options.json === true,
});
});
runtime
.command('prune')
.description('Remove stale managed Python runtimes for older CLI versions')
.option('--dry-run', 'List stale runtimes without deleting them', false)
.option('--yes', 'Confirm deletion of stale runtime directories', false)
.action(async (options: { dryRun?: boolean; yes?: boolean }) => {
await runRuntimeArgs(context, {
command: 'prune',
cliVersion: context.packageInfo.version,
dryRun: options.dryRun === true,
yes: options.yes === true,
});
});
}

320
packages/cli/src/commands/scan-commands.ts
View file

@ -1,5 +1,5 @@
import { type Command, InvalidArgumentError, Option } from '@commander-js/extra-typings';
import { type KtxCliCommandContext, parsePositiveIntegerOption, resolveCommandProjectDir } from '../cli-program.js';
import { type Command, InvalidArgumentError } from '@commander-js/extra-typings';
import { type KtxCliCommandContext, resolveCommandProjectDir } from '../cli-program.js';
import { runtimeInstallPolicyFromFlags } from '../managed-python-command.js';
import type { KtxScanArgs } from '../scan.js';
import { profileMark } from '../startup-profile.js';
@ -13,6 +13,16 @@ async function runScanArgs(context: KtxCliCommandContext, args: KtxScanArgs): Pr
type KtxScanModeOption = Extract<KtxScanArgs, { command: 'run' }>['mode'];
const REMOVED_SCAN_SUBCOMMAND_NAMES = new Set([
'status',
'report',
'relationships',
'relationship-apply',
'relationship-feedback',
'relationship-calibration',
'relationship-thresholds',
]);
function parseScanModeOption(value: string): KtxScanModeOption {
if (value === 'structural' || value === 'enriched' || value === 'relationships') {
return value;
@ -20,82 +30,18 @@ function parseScanModeOption(value: string): KtxScanModeOption {
throw new InvalidArgumentError('Allowed choices are structural, enriched, relationships');
}
type KtxRelationshipStatusOption = Extract<KtxScanArgs, { command: 'relationships' }>['status'];
type KtxRelationshipFeedbackDecisionOption = Extract<KtxScanArgs, { command: 'relationshipFeedback' }>['decision'];
function parseRelationshipStatusOption(value: string): KtxRelationshipStatusOption {
if (value === 'accepted' || value === 'review' || value === 'rejected' || value === 'skipped' || value === 'all') {
return value;
}
throw new InvalidArgumentError('Allowed choices are accepted, review, rejected, skipped, all');
}
function parseRelationshipFeedbackDecisionOption(value: string): KtxRelationshipFeedbackDecisionOption {
if (value === 'accepted' || value === 'rejected' || value === 'all') {
return value;
}
throw new InvalidArgumentError('Allowed choices are accepted, rejected, all');
}
function parseNonEmptyOption(value: string): string {
if (value.trim().length === 0) {
throw new InvalidArgumentError('must not be empty');
function parseConnectionId(value: string): string {
if (REMOVED_SCAN_SUBCOMMAND_NAMES.has(value)) {
throw new InvalidArgumentError(`"${value}" is not a scan connection id`);
}
return value;
}
function parseRelationshipCalibrationThreshold(value: string): number {
const parsed = Number(value);
if (Number.isFinite(parsed) && parsed >= 0 && parsed <= 1) {
return parsed;
}
throw new InvalidArgumentError('Allowed range is 0 through 1');
}
function relationshipDecisionArgs(options: {
accept?: string;
reject?: string;
reviewer?: string;
note?: string;
json?: boolean;
}): Pick<
Extract<KtxScanArgs, { command: 'relationshipDecision' }>,
'candidateId' | 'decision' | 'reviewer' | 'note' | 'json'
> | null {
const decisionCount = [options.accept !== undefined, options.reject !== undefined].filter(Boolean).length;
if (decisionCount > 1) {
throw new Error('Only one relationship review decision option can be used: --accept and --reject conflict');
}
if (options.accept !== undefined) {
return {
candidateId: options.accept,
decision: 'accepted',
reviewer: options.reviewer ?? 'ktx',
note: options.note ?? null,
json: options.json === true,
};
}
if (options.reject !== undefined) {
return {
candidateId: options.reject,
decision: 'rejected',
reviewer: options.reviewer ?? 'ktx',
note: options.note ?? null,
json: options.json === true,
};
}
return null;
}
function collectRelationshipCandidateOption(value: string, previous: string[]): string[] {
return [...previous, parseNonEmptyOption(value)];
}
export function registerScanCommands(program: Command, context: KtxCliCommandContext): void {
const scan = program
program
.command('scan')
.description('Run or inspect standalone connection scans')
.argument('[connectionId]', 'KTX connection id to scan')
.description('Run a standalone connection scan')
.argument('<connectionId>', 'KTX connection id to scan', parseConnectionId)
.option(
'--mode <mode>',
'Scan mode: structural, enriched, relationships (default: structural)',
@ -113,13 +59,7 @@ export function registerScanCommands(program: Command, context: KtxCliCommandCon
.hook('preAction', (_thisCommand, actionCommand) => {
context.writeDebug?.('scan', actionCommand);
})
.action(async (connectionId: string | undefined, options, command) => {
if (!connectionId) {
scan.outputHelp();
context.io.stderr.write('ktx dev scan requires <connectionId> or a subcommand\n');
context.setExitCode(1);
return;
}
.action(async (connectionId: string, options, command) => {
const mode = options.mode ?? 'structural';
await runScanArgs(context, {
command: 'run',
@ -133,226 +73,4 @@ export function registerScanCommands(program: Command, context: KtxCliCommandCon
runtimeInstallPolicy: runtimeInstallPolicyFromFlags(options),
});
});
scan
.command('status')
.description('Print status for a local scan run')
.argument('<runId>', 'Local scan run id')
.addHelpText(
'after',
'\n--project-dir is inherited from `ktx dev scan` (default: KTX_PROJECT_DIR or current working directory).\n',
)
.action(async (runId: string, _options: unknown, command) => {
await runScanArgs(context, {
command: 'status',
projectDir: resolveCommandProjectDir(command),
runId,
});
});
scan
.command('report')
.description('Print a local scan report')
.argument('<runId>', 'Local scan run id')
.option('--json', 'Print the raw scan report JSON', false)
.addHelpText(
'after',
'\n--project-dir is inherited from `ktx dev scan` (default: KTX_PROJECT_DIR or current working directory).\n',
)
.action(async (runId: string, options, command) => {
await runScanArgs(context, {
command: 'report',
projectDir: resolveCommandProjectDir(command),
runId,
json: options.json === true,
});
});
scan
.command('relationships')
.description('Print relationship artifacts for a local scan run')
.argument('<runId>', 'Local scan run id')
.option(
'--status <status>',
'Relationship status: accepted, review, rejected, skipped, all',
parseRelationshipStatusOption,
'review',
)
.option('--limit <count>', 'Maximum relationships to print per status', parsePositiveIntegerOption, 25)
.addOption(
new Option('--accept <candidateId>', 'Record a reviewer accepted decision for a relationship candidate')
.argParser(parseNonEmptyOption)
.conflicts('reject'),
)
.addOption(
new Option('--reject <candidateId>', 'Record a reviewer rejected decision for a relationship candidate')
.argParser(parseNonEmptyOption)
.conflicts('accept'),
)
.option('--note <text>', 'Attach a note when recording a relationship review decision')
.option('--reviewer <name>', 'Reviewer name for a relationship review decision')
.option('--json', 'Print relationship artifacts as JSON', false)
.addHelpText(
'after',
'\n--project-dir is inherited from `ktx dev scan` (default: KTX_PROJECT_DIR or current working directory).\n',
)
.action(async (runId: string, options, command) => {
const decision = relationshipDecisionArgs(options);
if (decision) {
await runScanArgs(context, {
command: 'relationshipDecision',
projectDir: resolveCommandProjectDir(command),
runId,
candidateId: decision.candidateId,
decision: decision.decision,
reviewer: decision.reviewer,
note: decision.note,
json: decision.json,
});
return;
}
await runScanArgs(context, {
command: 'relationships',
projectDir: resolveCommandProjectDir(command),
runId,
status: options.status,
json: options.json === true,
limit: options.limit,
});
});
scan
.command('relationship-apply')
.description('Apply accepted relationship review decisions as manual manifest joins')
.argument('<runId>', 'Local scan run id')
.option('--all-accepted', 'Apply all accepted relationship review decisions for the scan run', false)
.option(
'--candidate <candidateId>',
'Apply one accepted relationship review decision',
collectRelationshipCandidateOption,
[],
)
.option('--dry-run', 'Preview relationships that would be written without rewriting manifest shards', false)
.option('--json', 'Print the apply result as JSON', false)
.addHelpText(
'after',
'\n--project-dir is inherited from `ktx dev scan` (default: KTX_PROJECT_DIR or current working directory).\n',
)
.action(async (runId: string, options, command) => {
const parentOptions = command.parent?.opts() as { dryRun?: boolean } | undefined;
await runScanArgs(context, {
command: 'relationshipApply',
projectDir: resolveCommandProjectDir(command),
runId,
applyAllAccepted: options.allAccepted === true,
candidateIds: options.candidate,
dryRun: options.dryRun === true || parentOptions?.dryRun === true,
json: options.json === true,
});
});
scan
.command('relationship-feedback')
.description('Export persisted relationship review decisions as calibration labels')
.option('--connection <connectionId>', 'Only export labels for one KTX connection')
.option(
'--decision <decision>',
'Relationship feedback decision: accepted, rejected, all',
parseRelationshipFeedbackDecisionOption,
'all',
)
.addOption(new Option('--json', 'Print the export as JSON').default(false).conflicts('jsonl'))
.addOption(new Option('--jsonl', 'Print labels as newline-delimited JSON').default(false).conflicts('json'))
.addHelpText(
'after',
'\n--project-dir is inherited from `ktx dev scan` (default: KTX_PROJECT_DIR or current working directory).\n',
)
.action(async (options, command) => {
await runScanArgs(context, {
command: 'relationshipFeedback',
projectDir: resolveCommandProjectDir(command),
connectionId: options.connection ?? null,
decision: options.decision,
json: options.json === true,
jsonl: options.jsonl === true,
});
});
scan
.command('relationship-calibration')
.description('Summarize relationship feedback labels against current score thresholds')
.option('--connection <connectionId>', 'Only calibrate labels for one KTX connection')
.option(
'--decision <decision>',
'Relationship feedback decision: accepted, rejected, all',
parseRelationshipFeedbackDecisionOption,
'all',
)
.option(
'--accept-threshold <value>',
'Score threshold treated as predicted accepted',
parseRelationshipCalibrationThreshold,
0.85,
)
.option(
'--review-threshold <value>',
'Score threshold treated as predicted review',
parseRelationshipCalibrationThreshold,
0.55,
)
.option('--json', 'Print the calibration report as JSON', false)
.addHelpText(
'after',
'\n--project-dir is inherited from `ktx dev scan` (default: KTX_PROJECT_DIR or current working directory).\n',
)
.action(async (options, command) => {
await runScanArgs(context, {
command: 'relationshipCalibration',
projectDir: resolveCommandProjectDir(command),
connectionId: options.connection ?? null,
decision: options.decision,
acceptThreshold: options.acceptThreshold,
reviewThreshold: options.reviewThreshold,
json: options.json === true,
});
});
scan
.command('relationship-thresholds')
.description('Evaluate relationship feedback labels for offline threshold advice')
.option('--connection <connectionId>', 'Only evaluate labels for one KTX connection')
.option(
'--min-total-labels <count>',
'Minimum scored labels before advice can be ready',
parsePositiveIntegerOption,
20,
)
.option(
'--min-accepted-labels <count>',
'Minimum accepted labels before advice can be ready',
parsePositiveIntegerOption,
5,
)
.option(
'--min-rejected-labels <count>',
'Minimum rejected labels before advice can be ready',
parsePositiveIntegerOption,
5,
)
.option('--json', 'Print the threshold advice report as JSON', false)
.addHelpText(
'after',
'\n--project-dir is inherited from `ktx dev scan` (default: KTX_PROJECT_DIR or current working directory).\n',
)
.action(async (options, command) => {
await runScanArgs(context, {
command: 'relationshipThresholds',
projectDir: resolveCommandProjectDir(command),
connectionId: options.connection ?? null,
minTotalLabels: options.minTotalLabels,
minAcceptedLabels: options.minAcceptedLabels,
minRejectedLabels: options.minRejectedLabels,
json: options.json === true,
});
});
}

42
packages/cli/src/commands/setup-commands.ts
View file

@ -2,6 +2,7 @@ import { type Command, InvalidArgumentError, Option } from '@commander-js/extra-
import type { KtxCliCommandContext } from '../cli-program.js';
import { resolveCommandProjectDir } from '../cli-program.js';
import type { KtxSetupDatabaseDriver } from '../setup-databases.js';
import type { KtxSetupLlmBackend } from '../setup-models.js';
import type { KtxSetupSourceType } from '../setup-sources.js';
async function runSetupArgs(
@ -27,6 +28,13 @@ function embeddingBackend(value: string): 'openai' | 'sentence-transformers' {
throw new InvalidArgumentError(`invalid choice '${value}'`);
}
function llmBackend(value: string): KtxSetupLlmBackend {
if (value === 'anthropic' || value === 'vertex') {
return value;
}
throw new InvalidArgumentError(`invalid choice '${value}'`);
}
function databaseDriver(value: string): KtxSetupDatabaseDriver {
if (
value === 'sqlite' ||
@ -93,9 +101,12 @@ function shouldShowSetupEntryMenu(
skipAgents?: boolean;
yes?: boolean;
input?: boolean;
llmBackend?: KtxSetupLlmBackend;
anthropicApiKeyEnv?: string;
anthropicApiKeyFile?: string;
anthropicModel?: string;
vertexProject?: string;
vertexLocation?: string;
skipLlm?: boolean;
embeddingBackend?: string;
embeddingApiKeyEnv?: string;
@ -110,7 +121,6 @@ function shouldShowSetupEntryMenu(
disableHistoricSql?: boolean;
historicSqlWindowDays?: number;
historicSqlMinExecutions?: number;
historicSqlMinCalls?: number;
historicSqlServiceAccountPattern?: string[];
historicSqlRedactionPattern?: string[];
skipDatabases?: boolean;
@ -166,9 +176,12 @@ function shouldShowSetupEntryMenu(
'skipAgents',
'yes',
'input',
'llmBackend',
'anthropicApiKeyEnv',
'anthropicApiKeyFile',
'anthropicModel',
'vertexProject',
'vertexLocation',
'skipLlm',
'embeddingBackend',
'embeddingApiKeyEnv',
@ -180,7 +193,6 @@ function shouldShowSetupEntryMenu(
'disableHistoricSql',
'historicSqlWindowDays',
'historicSqlMinExecutions',
'historicSqlMinCalls',
'skipDatabases',
'source',
'sourceConnectionId',
@ -227,9 +239,12 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo
.option('--skip-agents', 'Leave agent integration incomplete for now', false)
.option('--yes', 'Accept safe defaults in non-interactive setup', false)
.option('--no-input', 'Disable interactive terminal input')
.addOption(new Option('--llm-backend <backend>', 'LLM backend').argParser(llmBackend))
.option('--anthropic-api-key-env <name>', 'Environment variable containing the Anthropic API key')
.option('--anthropic-api-key-file <path>', 'File containing the Anthropic API key')
.option('--anthropic-model <model>', 'Anthropic model ID to validate and save')
.option('--vertex-project <project>', 'Google Vertex AI project ID, env:NAME, or file:/path')
.option('--vertex-location <location>', 'Google Vertex AI location, env:NAME, or file:/path')
.addOption(new Option('--skip-llm', 'Leave LLM setup incomplete for now').hideHelp().default(false))
.addOption(new Option('--embedding-backend <backend>', 'Embedding backend').argParser(embeddingBackend))
.option('--embedding-api-key-env <name>', 'Environment variable containing the embedding provider API key')
@ -266,11 +281,6 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo
.option('--disable-historic-sql', 'Disable Historic SQL for the selected database', false)
.option('--historic-sql-window-days <number>', 'Historic SQL query-history window', positiveInteger)
.option('--historic-sql-min-executions <number>', 'Minimum Historic SQL executions for a template', positiveInteger)
.option(
'--historic-sql-min-calls <number>',
'Alias for --historic-sql-min-executions',
positiveInteger,
)
.option(
'--historic-sql-service-account-pattern <pattern>',
'Historic SQL service-account regex; repeatable',
@ -344,6 +354,16 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo
context.setExitCode(1);
return;
}
if (options.llmBackend === 'vertex' && (options.anthropicApiKeyEnv || options.anthropicApiKeyFile)) {
context.io.stderr.write('Anthropic API key flags are only valid with --llm-backend anthropic.\n');
context.setExitCode(1);
return;
}
if (options.llmBackend === 'anthropic' && (options.vertexProject || options.vertexLocation)) {
context.io.stderr.write('Vertex AI flags are only valid with --llm-backend vertex.\n');
context.setExitCode(1);
return;
}
if (options.embeddingApiKeyEnv && options.embeddingApiKeyFile) {
context.io.stderr.write(
'Choose only one embedding credential source: --embedding-api-key-env or --embedding-api-key-file.\n',
@ -371,7 +391,6 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo
const mode = options.new ? 'new' : options.existing ? 'existing' : 'auto';
const resolvedAgentScope = options.global ? 'global' : options.agentScope;
const historicSqlMinExecutions = options.historicSqlMinExecutions ?? options.historicSqlMinCalls;
await runSetupArgs(context, {
command: 'run',
projectDir: resolveCommandProjectDir(command),
@ -383,9 +402,12 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo
inputMode: options.input === false ? 'disabled' : 'auto',
yes: options.yes === true,
cliVersion: context.packageInfo.version,
...(options.llmBackend ? { llmBackend: options.llmBackend } : {}),
...(options.anthropicApiKeyEnv ? { anthropicApiKeyEnv: options.anthropicApiKeyEnv } : {}),
...(options.anthropicApiKeyFile ? { anthropicApiKeyFile: options.anthropicApiKeyFile } : {}),
...(options.anthropicModel ? { anthropicModel: options.anthropicModel } : {}),
...(options.vertexProject ? { vertexProject: options.vertexProject } : {}),
...(options.vertexLocation ? { vertexLocation: options.vertexLocation } : {}),
skipLlm: options.skipLlm === true,
...(options.embeddingBackend ? { embeddingBackend: options.embeddingBackend } : {}),
...(options.embeddingApiKeyEnv ? { embeddingApiKeyEnv: options.embeddingApiKeyEnv } : {}),
@ -399,7 +421,9 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo
...(options.enableHistoricSql ? { enableHistoricSql: true } : {}),
...(options.disableHistoricSql ? { disableHistoricSql: true } : {}),
...(options.historicSqlWindowDays !== undefined ? { historicSqlWindowDays: options.historicSqlWindowDays } : {}),
...(historicSqlMinExecutions !== undefined ? { historicSqlMinExecutions } : {}),
...(options.historicSqlMinExecutions !== undefined
? { historicSqlMinExecutions: options.historicSqlMinExecutions }
: {}),
...(options.historicSqlServiceAccountPattern.length > 0
? { historicSqlServiceAccountPatterns: options.historicSqlServiceAccountPattern }
: {}),

104
packages/cli/src/commands/sl-commands.ts
View file

@ -41,7 +41,7 @@ async function runSlArgs(context: KtxCliCommandContext, args: KtxSlArgs): Promis
export function registerSlCommands(program: Command, context: KtxCliCommandContext, commandName = 'sl'): void {
const sl = program
.command(commandName)
.description('List, read, validate, query, or write local semantic-layer sources')
.description('List, search, validate, or query local semantic-layer sources')
.showHelpAfterError()
.addHelpText(
'after',
@ -59,28 +59,48 @@ export function registerSlCommands(program: Command, context: KtxCliCommandConte
]),
)
.option('--json', 'Shortcut for --output=json (overrides --output)', false)
.action(async (options: { connectionId?: string; output?: 'pretty' | 'plain' | 'json'; json?: boolean }, command) => {
await runSlArgs(context, {
command: 'list',
projectDir: resolveCommandProjectDir(command),
connectionId: options.connectionId,
output: options.output,
json: options.json,
});
});
.action(
async (options: { connectionId?: string; output?: 'pretty' | 'plain' | 'json'; json?: boolean }, command) => {
await runSlArgs(context, {
command: 'list',
projectDir: resolveCommandProjectDir(command),
connectionId: options.connectionId,
output: options.output,
json: options.json,
});
},
);
sl.command('read')
.description('Read a semantic-layer source')
.argument('<sourceName>', 'Semantic-layer source name')
.requiredOption('--connection-id <id>', 'KTX connection id')
.action(async (sourceName: string, options: { connectionId: string }, command) => {
await runSlArgs(context, {
command: 'read',
projectDir: resolveCommandProjectDir(command),
connectionId: options.connectionId,
sourceName,
});
});
sl.command('search')
.description('Search semantic-layer sources')
.argument('<query>', 'Search query')
.option('--connection-id <id>', 'KTX connection id')
.option('--limit <number>', 'Maximum search results', parsePositiveIntegerOption)
.addOption(
new Option('--output <mode>', 'Output mode: pretty (default in TTY), plain (TSV), or json').choices([
'pretty',
'plain',
'json',
]),
)
.option('--json', 'Shortcut for --output=json (overrides --output)', false)
.action(
async (
query: string,
options: { connectionId?: string; limit?: number; output?: 'pretty' | 'plain' | 'json'; json?: boolean },
command,
) => {
await runSlArgs(context, {
command: 'search',
projectDir: resolveCommandProjectDir(command),
connectionId: options.connectionId,
query,
...(options.limit !== undefined ? { limit: options.limit } : {}),
output: options.output,
json: options.json,
});
},
);
sl.command('validate')
.description('Validate a semantic-layer source')
@ -95,24 +115,10 @@ export function registerSlCommands(program: Command, context: KtxCliCommandConte
});
});
sl.command('write')
.description('Write a semantic-layer source')
.argument('<sourceName>', 'Semantic-layer source name')
.requiredOption('--connection-id <id>', 'KTX connection id')
.requiredOption('--yaml <yaml>', 'Semantic-layer source YAML')
.action(async (sourceName: string, options: { connectionId: string; yaml: string }, command) => {
await runSlArgs(context, {
command: 'write',
projectDir: resolveCommandProjectDir(command),
connectionId: options.connectionId,
sourceName,
yaml: options.yaml,
});
});
sl.command('query')
.description('Compile or execute a semantic-layer query')
.option('--connection-id <id>', 'KTX connection id')
.option('--query-file <path>', 'JSON semantic-layer query file')
.option('--measure <measure>', 'Measure to query; repeatable', collectOption, [])
.option('--dimension <dimension>', 'Dimension to include; repeatable', collectOption, [])
.option('--filter <filter>', 'Filter expression; repeatable', collectOption, [])
@ -126,22 +132,26 @@ export function registerSlCommands(program: Command, context: KtxCliCommandConte
.option('--no-input', 'Disable interactive managed runtime installation')
.option('--max-rows <n>', 'Maximum rows to return when executing', parsePositiveIntegerOption)
.action(async (options, command) => {
if (options.measure.length === 0) {
if (options.measure.length === 0 && !options.queryFile) {
throw new Error('sl query requires at least one --measure');
}
const args = slQueryCommandSchema.parse({
command: 'query',
projectDir: resolveCommandProjectDir(command),
connectionId: options.connectionId,
query: {
measures: options.measure,
dimensions: options.dimension,
...(options.filter.length > 0 ? { filters: options.filter } : {}),
...(options.segment.length > 0 ? { segments: options.segment } : {}),
...(options.orderBy.length > 0 ? { order_by: options.orderBy } : {}),
...(options.limit !== undefined ? { limit: options.limit } : {}),
...(options.includeEmpty === true ? { include_empty: true } : {}),
},
...(options.queryFile
? { queryFile: options.queryFile }
: {
query: {
measures: options.measure,
dimensions: options.dimension,
...(options.filter.length > 0 ? { filters: options.filter } : {}),
...(options.segment.length > 0 ? { segments: options.segment } : {}),
...(options.orderBy.length > 0 ? { order_by: options.orderBy } : {}),
...(options.limit !== undefined ? { limit: options.limit } : {}),
...(options.includeEmpty === true ? { include_empty: true } : {}),
},
}),
format: options.format,
execute: options.execute === true,
cliVersion: context.packageInfo.version,

353
packages/cli/src/completion.ts
View file

@ -1,353 +0,0 @@
import { mkdir, readFile, writeFile } from 'node:fs/promises';
import { homedir } from 'node:os';
import { dirname, join } from 'node:path';
import type { CommandUnknownOpts, Option } from '@commander-js/extra-typings';
export interface CompletionRequest {
position: number;
words: string[];
}
interface CompletionCandidate {
value: string;
description?: string;
}
interface CommandWithHiddenFlag extends CommandUnknownOpts {
_hidden?: boolean;
}
interface ResolveState {
command: CommandUnknownOpts;
pendingOption?: Option;
positionalIndex: number;
}
export interface ZshCompletionInstallResult {
completionPath: string;
zshrcPath: string;
}
const KTX_COMPLETION_BLOCK_START = '# >>> ktx completion >>>';
const KTX_COMPLETION_BLOCK_END = '# <<< ktx completion <<<';
const KTX_COMPLETION_BLOCK_PATTERN = new RegExp(
`\\n?${escapeRegExp(KTX_COMPLETION_BLOCK_START)}[\\s\\S]*?${escapeRegExp(KTX_COMPLETION_BLOCK_END)}\\n?`,
'g',
);
export function zshCompletionScript(): string {
const zshWords = '$' + '{words[@]}';
const zshCompletionCapture = [
'$',
`{(@f)$("${'$'}{ktx_completion_command[@]}" dev __complete --shell zsh --position "$CURRENT" -- "${zshWords}" 2>/dev/null)}`,
].join('');
const zshCompletionsCount = '$' + '{#completions[@]}';
const zshCompletionCommand = '$' + '(eval "print -r -- $' + '{KTX_COMPLETION_COMMAND:-ktx}")';
return [
'#compdef ktx',
'',
'_ktx() {',
' local -a completions',
' local -a ktx_completion_command',
` ktx_completion_command=("\${(@z)${zshCompletionCommand}}")`,
` completions=("${zshCompletionCapture}")`,
` if (( ${zshCompletionsCount} )); then`,
" _describe 'ktx completions' completions",
' else',
' _files',
' fi',
'}',
'',
'compdef _ktx ktx',
'',
].join('\n');
}
export async function installZshCompletion(): Promise<ZshCompletionInstallResult> {
const homeDir = process.env.HOME || homedir();
const zshConfigDir = process.env.ZDOTDIR || homeDir;
const completionDir = join(homeDir, '.zfunc');
const completionPath = join(completionDir, '_ktx');
const zshrcPath = join(zshConfigDir, '.zshrc');
await mkdir(completionDir, { recursive: true });
await mkdir(dirname(zshrcPath), { recursive: true });
await writeFile(completionPath, zshCompletionScript(), 'utf-8');
const existingZshrc = await readOptionalTextFile(zshrcPath);
const nextZshrc = updateZshrcCompletionBlock(existingZshrc);
await writeFile(zshrcPath, nextZshrc, 'utf-8');
return { completionPath, zshrcPath };
}
export function completeCommanderInput(program: CommandUnknownOpts, request: CompletionRequest): string[] {
const words = completionWordsForPosition(request.words, request.position);
const tokens = stripProgramName(program, words);
const current = tokens.at(-1) ?? '';
const previous = tokens.slice(0, -1);
const state = resolveCommandState(program, previous);
return candidatesForState(state, current).map(formatZshCandidate);
}
function completionWordsForPosition(words: string[], position: number): string[] {
if (!Number.isInteger(position) || position < 1) {
return words;
}
return words.slice(0, position);
}
function stripProgramName(program: CommandUnknownOpts, words: string[]): string[] {
const [first, ...rest] = words;
if (!first) {
return [];
}
return first === program.name() || first.endsWith(`/${program.name()}`) ? rest : words;
}
function resolveCommandState(program: CommandUnknownOpts, tokens: string[]): ResolveState {
let command = program;
let positionalIndex = 0;
let pendingOption: Option | undefined;
let positionalOnly = false;
for (let index = 0; index < tokens.length; index += 1) {
const token = tokens[index];
if (pendingOption) {
pendingOption = undefined;
continue;
}
if (token === '--') {
positionalOnly = true;
continue;
}
if (!positionalOnly && token.startsWith('-')) {
const option = findOption(command, optionNameFromToken(token));
if (option && !token.includes('=') && optionTakesValue(option)) {
if (index === tokens.length - 1) {
pendingOption = option;
} else if (option.required || !tokens[index + 1]?.startsWith('-')) {
index += 1;
}
}
continue;
}
const child = findVisibleSubcommand(command, token);
if (child) {
command = child;
positionalIndex = 0;
continue;
}
positionalIndex += 1;
}
return { command, pendingOption, positionalIndex };
}
function candidatesForState(state: ResolveState, current: string): CompletionCandidate[] {
const optionValue = splitOptionValueToken(current);
if (optionValue) {
const option = findOption(state.command, optionValue.optionName);
return choiceCandidates(option?.argChoices, optionValue.valuePrefix, optionValue.optionPrefix);
}
if (state.pendingOption) {
return choiceCandidates(state.pendingOption.argChoices, current);
}
if (current.startsWith('-')) {
return visibleOptions(state.command)
.map(optionCandidate)
.filter((candidate) => candidate.value.startsWith(current));
}
const commandCandidates = visibleSubcommands(state.command)
.map(commandCandidate)
.filter((candidate) => candidate.value.startsWith(current));
const argument = state.command.registeredArguments[state.positionalIndex];
return [...commandCandidates, ...choiceCandidates(argument?.argChoices, current)];
}
function visibleSubcommands(command: CommandUnknownOpts): CommandUnknownOpts[] {
return command.commands.filter((subcommand) => (subcommand as CommandWithHiddenFlag)._hidden !== true);
}
function findVisibleSubcommand(command: CommandUnknownOpts, name: string): CommandUnknownOpts | undefined {
return visibleSubcommands(command).find(
(subcommand) => subcommand.name() === name || subcommand.aliases().includes(name),
);
}
function visibleOptions(command: CommandUnknownOpts): Option[] {
const options: Option[] = [];
const seen = new Set<string>();
for (const current of commandChain(command)) {
for (const option of current.options) {
if (option.hidden) {
continue;
}
const key = option.long ?? option.short ?? option.flags;
if (seen.has(key)) {
continue;
}
seen.add(key);
options.push(option);
}
}
return options;
}
function commandChain(command: CommandUnknownOpts): CommandUnknownOpts[] {
const chain: CommandUnknownOpts[] = [];
let current: CommandUnknownOpts | null = command;
while (current) {
chain.unshift(current);
current = current.parent;
}
return chain;
}
function findOption(command: CommandUnknownOpts, name: string): Option | undefined {
return visibleOptions(command).find((option) => option.long === name || option.short === name);
}
function optionTakesValue(option: Option): boolean {
return option.required || option.optional;
}
function optionNameFromToken(token: string): string {
return token.split('=', 1)[0] ?? token;
}
function splitOptionValueToken(
token: string,
): { optionName: string; optionPrefix: string; valuePrefix: string } | null {
const separatorIndex = token.indexOf('=');
if (!token.startsWith('-') || separatorIndex < 0) {
return null;
}
return {
optionName: token.slice(0, separatorIndex),
optionPrefix: token.slice(0, separatorIndex + 1),
valuePrefix: token.slice(separatorIndex + 1),
};
}
function commandCandidate(command: CommandUnknownOpts): CompletionCandidate {
return {
value: command.name(),
description: command.summary() || command.description(),
};
}
function optionCandidate(option: Option): CompletionCandidate {
return {
value: option.long ?? option.short ?? option.flags,
description: option.description,
};
}
function choiceCandidates(
choices: readonly string[] | undefined,
prefix: string,
completionPrefix = '',
): CompletionCandidate[] {
return (choices ?? [])
.filter((choice) => choice.startsWith(prefix))
.map((choice) => ({ value: `${completionPrefix}${choice}` }));
}
function formatZshCandidate(candidate: CompletionCandidate): string {
if (!candidate.description) {
return escapeZshCompletion(candidate.value);
}
return `${escapeZshCompletion(candidate.value)}:${escapeZshDescription(candidate.description)}`;
}
function escapeZshCompletion(value: string): string {
return value.replace(/\\/g, '\\\\').replace(/:/g, '\\:');
}
function escapeZshDescription(value: string): string {
return value.replace(/\s+/g, ' ').replace(/\\/g, '\\\\').replace(/:/g, '\\:').trim();
}
async function readOptionalTextFile(path: string): Promise<string> {
try {
return await readFile(path, 'utf-8');
} catch (error) {
if (isNodeError(error) && error.code === 'ENOENT') {
return '';
}
throw error;
}
}
function updateZshrcCompletionBlock(contents: string): string {
const withoutManagedBlock = contents.replace(KTX_COMPLETION_BLOCK_PATTERN, normalizeTrailingNewline);
const hasCompinit = /^.*\bcompinit\b.*$/m.test(withoutManagedBlock);
const block = zshrcCompletionBlock({ includeCompinit: !hasCompinit });
if (!hasCompinit) {
return appendBlock(withoutManagedBlock, block);
}
const compinitMatch = /^.*\bcompinit\b.*$/m.exec(withoutManagedBlock);
if (!compinitMatch || compinitMatch.index === undefined) {
return appendBlock(withoutManagedBlock, block);
}
return [
withoutManagedBlock.slice(0, compinitMatch.index),
block,
'\n',
withoutManagedBlock.slice(compinitMatch.index),
].join('');
}
function zshrcCompletionBlock(options: { includeCompinit: boolean }): string {
return [
KTX_COMPLETION_BLOCK_START,
'_ktx_completion_command() {',
' local dir="$PWD"',
' while [[ "$dir" != "/" ]]; do',
` if [[ -f "$dir/package.json" ]] && command grep -q '"name": "ktx-workspace"' "$dir/package.json" 2>/dev/null; then`,
' print -r -- "node $dir/scripts/run-ktx.mjs --"',
' return',
' fi',
' dir="' + '$' + '{dir:h}"',
' done',
' print -r -- "ktx"',
'}',
"export KTX_COMPLETION_COMMAND='$(_ktx_completion_command)'",
'setopt complete_aliases',
'fpath=("$HOME/.zfunc" $fpath)',
...(options.includeCompinit ? ['autoload -Uz compinit', 'compinit'] : []),
KTX_COMPLETION_BLOCK_END,
].join('\n');
}
function appendBlock(contents: string, block: string): string {
if (!contents.trim()) {
return `${block}\n`;
}
return `${contents.replace(/\s*$/, '\n\n')}${block}\n`;
}
function normalizeTrailingNewline(match: string): string {
return match.startsWith('\n') || match.endsWith('\n') ? '\n' : '';
}
function escapeRegExp(value: string): string {
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
}
function isNodeError(error: unknown): error is NodeJS.ErrnoException {
return error instanceof Error && 'code' in error;
}

541
packages/cli/src/connection.test.ts
View file

@ -1,4 +1,4 @@
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import type { MetabaseRuntimeClient } from '@ktx/context/ingest';
@ -6,18 +6,13 @@ import { initKtxProject, parseKtxProjectConfig, serializeKtxProjectConfig } from
import type { KtxConnectionDriver, KtxScanConnector, KtxSchemaSnapshot } from '@ktx/context/scan';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { runKtxConnection } from './connection.js';
import { runKtxCli, type KtxCliIo } from './index.js';
function makeIo(options: { stdoutIsTty?: boolean; stdinIsTty?: boolean } = {}) {
function makeIo() {
let stdout = '';
let stderr = '';
return {
io: {
stdin: {
isTTY: options.stdinIsTty,
},
stdout: {
isTTY: options.stdoutIsTty,
write: (chunk: string) => {
stdout += chunk;
},
@ -87,491 +82,49 @@ describe('runKtxConnection', () => {
await rm(tempDir, { recursive: true, force: true });
});
it('adds and lists env-referenced connections without resolving secrets', async () => {
async function writeConnections(
projectDir: string,
connections: ReturnType<typeof parseKtxProjectConfig>['connections'],
): Promise<void> {
const config = parseKtxProjectConfig(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'));
await writeFile(join(projectDir, 'ktx.yaml'), serializeKtxProjectConfig({ ...config, connections }), 'utf-8');
}
it('lists configured connections without resolving secrets', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
await writeConnections(projectDir, {
warehouse: { driver: 'postgres', url: 'env:DATABASE_URL', readonly: true },
docs: { driver: 'notion', auth_token_ref: 'env:NOTION_TOKEN', crawl_mode: 'all_accessible' },
});
const io = makeIo();
await expect(
runKtxConnection(
{
command: 'add',
projectDir,
driver: 'postgres',
connectionId: 'warehouse',
url: 'env:DATABASE_URL',
schemas: ['public'],
readonly: true,
force: false,
allowLiteralCredentials: false,
},
io.io,
),
).resolves.toBe(0);
await expect(runKtxConnection({ command: 'list', projectDir }, io.io)).resolves.toBe(0);
expect(io.stdout()).toContain('Connection: warehouse');
await expect(readFile(join(projectDir, 'ktx.yaml'), 'utf-8')).resolves.toContain('url: env:DATABASE_URL');
const listIo = makeIo();
await expect(runKtxConnection({ command: 'list', projectDir }, listIo.io)).resolves.toBe(0);
expect(listIo.stdout()).toContain('warehouse');
expect(listIo.stdout()).toContain('postgres');
});
it('removes a configured connection from ktx.yaml without deleting local artifacts when forced', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
await runKtxConnection(
{
command: 'add',
projectDir,
driver: 'sqlite',
connectionId: 'warehouse',
url: undefined,
schemas: [],
readonly: true,
force: false,
allowLiteralCredentials: false,
},
makeIo().io,
);
const artifactPath = join(projectDir, '.ktx', 'artifacts', 'warehouse.txt');
await mkdir(join(projectDir, '.ktx', 'artifacts'), { recursive: true });
await writeFile(artifactPath, 'keep me', 'utf-8');
const io = makeIo();
await expect(
runKtxConnection(
{
command: 'remove',
projectDir,
connectionId: 'warehouse',
force: true,
inputMode: 'disabled',
},
io.io,
),
).resolves.toBe(0);
const parsed = parseKtxProjectConfig(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'));
expect(parsed.connections.warehouse).toBeUndefined();
await expect(readFile(artifactPath, 'utf-8')).resolves.toBe('keep me');
expect(io.stdout()).toContain('Connection removed from ktx.yaml.');
expect(io.stdout()).toContain(
'Ingested artifacts from this connection remain in .ktx/. Run ktx dev artifacts to inspect.',
);
expect(io.stdout()).toContain('warehouse');
expect(io.stdout()).toContain('postgres');
expect(io.stdout()).toContain('docs');
expect(io.stdout()).toContain('notion');
expect(io.stderr()).toBe('');
});
it('requires --force when removing in non-interactive mode', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
await runKtxConnection(
{
command: 'add',
projectDir,
driver: 'sqlite',
connectionId: 'warehouse',
url: undefined,
schemas: [],
readonly: true,
force: false,
allowLiteralCredentials: false,
},
makeIo().io,
);
const io = makeIo();
await expect(
runKtxConnection(
{
command: 'remove',
projectDir,
connectionId: 'warehouse',
force: false,
inputMode: 'disabled',
},
io.io,
),
).resolves.toBe(1);
expect(io.stderr()).toContain('connection remove warehouse requires --force when input is disabled or not interactive');
});
it('returns a clear error when removing an unknown connection', async () => {
it('prints an empty-state message that points at setup instead of removed connection add', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
const io = makeIo();
await expect(
runKtxConnection(
{
command: 'remove',
projectDir,
connectionId: 'missing',
force: true,
inputMode: 'disabled',
},
io.io,
),
).resolves.toBe(1);
await expect(runKtxConnection({ command: 'list', projectDir }, io.io)).resolves.toBe(0);
expect(io.stderr()).toContain('Connection "missing" is not configured in ktx.yaml');
});
it('asks for confirmation before removing in an interactive terminal', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
await runKtxConnection(
{
command: 'add',
projectDir,
driver: 'sqlite',
connectionId: 'warehouse',
url: undefined,
schemas: [],
readonly: true,
force: false,
allowLiteralCredentials: false,
},
makeIo().io,
);
const io = makeIo({ stdoutIsTty: true, stdinIsTty: true });
const prompts = {
confirm: vi.fn(async () => true),
cancel: vi.fn(),
};
await expect(
runKtxConnection(
{
command: 'remove',
projectDir,
connectionId: 'warehouse',
force: false,
},
io.io,
{ prompts },
),
).resolves.toBe(0);
expect(prompts.confirm).toHaveBeenCalledWith({
message: 'Remove connection "warehouse" from ktx.yaml? Ingested artifacts will remain in .ktx/.',
initialValue: false,
});
});
it('runs public connect map as refresh, validate, and list over the low-level mapping runner', async () => {
const io = makeIo();
const runMapping = vi.fn(async (argv: string[], mappingIo: KtxCliIo) => {
if (argv[0] === 'refresh') {
mappingIo.stdout.write('Discovery: 1 database\n');
mappingIo.stdout.write('Unmapped discovered: 1\n');
mappingIo.stdout.write('Stale mappings: 0\n');
return 0;
}
if (argv[0] === 'validate') {
mappingIo.stdout.write('Mapping validation passed: prod-metabase\n');
return 0;
}
if (argv[0] === 'list') {
mappingIo.stdout.write('1 -> [unmapped] (Analytics, sync: on, source: refresh)\n');
return 0;
}
return 1;
});
await expect(
runKtxConnection(
{ command: 'map', projectDir: '/tmp/project', sourceConnectionId: 'prod-metabase', json: false },
io.io,
{ runMapping },
),
).resolves.toBe(0);
expect(runMapping).toHaveBeenNthCalledWith(
1,
['refresh', 'prod-metabase', '--auto-accept', '--project-dir', '/tmp/project'],
expect.any(Object),
);
expect(runMapping).toHaveBeenNthCalledWith(
2,
['validate', 'prod-metabase', '--project-dir', '/tmp/project'],
expect.any(Object),
);
expect(runMapping).toHaveBeenNthCalledWith(
3,
['list', 'prod-metabase', '--project-dir', '/tmp/project'],
expect.any(Object),
);
expect(io.stdout()).toContain('Mapping: prod-metabase');
expect(io.stdout()).toContain('Discovery: 1 database');
expect(io.stdout()).toContain('Mappings:');
expect(io.stdout()).toContain('1 -> [unmapped]');
expect(io.stdout()).toContain('Next:');
expect(io.stdout()).toContain('ktx ingest prod-metabase');
expect(io.stdout()).toContain('ktx dev mapping');
expect(io.stderr()).toBe('');
});
it('prints stable JSON for public connect map without leaking low-level stdout', async () => {
const io = makeIo();
const runMapping = vi.fn(async (argv: string[], mappingIo: KtxCliIo) => {
if (argv[0] === 'refresh') {
mappingIo.stdout.write('Discovery: 1 connection\nUnmapped discovered: 0\nStale mappings: 0\n');
return 0;
}
if (argv[0] === 'validate') {
mappingIo.stdout.write('Mapping validation passed: prod-looker\n');
return 0;
}
if (argv[0] === 'list') {
expect(argv).toContain('--json');
mappingIo.stdout.write(
`${JSON.stringify(
[
{
lookerConnectionName: 'analytics',
ktxConnectionId: 'prod-warehouse',
source: 'ktx.yaml',
},
],
null,
2,
)}\n`,
);
return 0;
}
return 1;
});
await expect(
runKtxConnection(
{ command: 'map', projectDir: '/tmp/project', sourceConnectionId: 'prod-looker', json: true },
io.io,
{ runMapping },
),
).resolves.toBe(0);
const parsed = JSON.parse(io.stdout()) as {
connectionId: string;
refresh: { ok: boolean; output: string[] };
validation: { ok: boolean; output: string[] };
mappings: Array<{ lookerConnectionName: string; ktxConnectionId: string; source: string }>;
};
expect(parsed).toEqual({
connectionId: 'prod-looker',
refresh: {
ok: true,
output: ['Discovery: 1 connection', 'Unmapped discovered: 0', 'Stale mappings: 0'],
},
validation: {
ok: true,
output: ['Mapping validation passed: prod-looker'],
},
mappings: [
{
lookerConnectionName: 'analytics',
ktxConnectionId: 'prod-warehouse',
source: 'ktx.yaml',
},
],
});
expect(io.stderr()).toBe('');
});
it('returns the refresh failure when public connect map cannot discover source metadata', async () => {
const io = makeIo();
const runMapping = vi.fn(async (argv: string[], mappingIo: KtxCliIo) => {
if (argv[0] === 'refresh') {
mappingIo.stderr.write('Metabase API key is not configured\n');
return 1;
}
return 0;
});
await expect(
runKtxConnection(
{ command: 'map', projectDir: '/tmp/project', sourceConnectionId: 'prod-metabase', json: false },
io.io,
{ runMapping },
),
).resolves.toBe(1);
expect(runMapping).toHaveBeenCalledTimes(1);
expect(io.stdout()).toBe('');
expect(io.stderr()).toContain('Metabase API key is not configured');
});
it('rejects literal credential URLs unless explicitly allowed', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
const io = makeIo();
await expect(
runKtxConnection(
{
command: 'add',
projectDir,
driver: 'postgres',
connectionId: 'warehouse',
url: 'postgres://localhost:5432/warehouse',
schemas: [],
readonly: true,
force: false,
allowLiteralCredentials: false,
},
io.io,
),
).resolves.toBe(1);
expect(io.stderr()).toContain('Literal credential URLs require --allow-literal-credentials');
});
it('warns before writing explicitly allowed literal credential URLs without echoing the URL', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
const io = makeIo();
const literalUrl = 'postgres://localhost:5432/warehouse';
await expect(
runKtxConnection(
{
command: 'add',
projectDir,
driver: 'postgres',
connectionId: 'warehouse',
url: literalUrl,
schemas: ['public'],
readonly: true,
force: false,
allowLiteralCredentials: true,
},
io.io,
),
).resolves.toBe(0);
expect(io.stderr()).toContain(
'Warning: writing a literal credential URL to ktx.yaml for connection "warehouse". Prefer env:NAME or file:/path references.',
);
expect(io.stderr()).not.toContain(literalUrl);
await expect(readFile(join(projectDir, 'ktx.yaml'), 'utf-8')).resolves.toContain(literalUrl);
});
it('adds a Notion connection without writing token values', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
const io = makeIo();
await expect(
runKtxConnection(
{
command: 'add',
projectDir,
driver: 'notion',
connectionId: 'notion-main',
url: undefined,
schemas: [],
readonly: false,
force: false,
allowLiteralCredentials: false,
notion: {
authTokenRef: 'env:NOTION_TOKEN',
crawlMode: 'all_accessible',
rootPageIds: [],
rootDatabaseIds: [],
rootDataSourceIds: [],
maxPagesPerRun: 50,
maxKnowledgeCreatesPerRun: 4,
maxKnowledgeUpdatesPerRun: 12,
},
},
io.io,
),
).resolves.toBe(0);
const yaml = await readFile(join(projectDir, 'ktx.yaml'), 'utf-8');
expect(yaml).toContain('driver: notion');
expect(yaml).toContain('auth_token_ref: env:NOTION_TOKEN');
expect(yaml).toContain('crawl_mode: all_accessible');
expect(yaml).toContain('max_pages_per_run: 50');
expect(yaml).not.toContain('ntn_');
expect(io.stdout()).toContain('Connection: notion-main');
expect(io.stdout()).toContain('Driver: notion');
});
it('runs connection notion pick --no-input through the public connection entrypoint', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
await runKtxConnection(
{
command: 'add',
projectDir,
driver: 'notion',
connectionId: 'notion-main',
url: undefined,
schemas: [],
readonly: false,
force: false,
allowLiteralCredentials: false,
notion: {
authTokenRef: 'env:NOTION_TOKEN',
crawlMode: 'all_accessible',
rootPageIds: [],
rootDatabaseIds: ['database-1'],
rootDataSourceIds: ['data-source-1'],
maxPagesPerRun: 50,
maxKnowledgeCreatesPerRun: 4,
maxKnowledgeUpdatesPerRun: 12,
},
},
makeIo().io,
);
const io = makeIo();
await expect(
runKtxCli(
[
'connection',
'notion',
'pick',
'notion-main',
'--project-dir',
projectDir,
'--no-input',
'--root-page-id',
'11111111222233334444555555555555',
],
io.io,
),
).resolves.toBe(0);
const yaml = await readFile(join(projectDir, 'ktx.yaml'), 'utf-8');
expect(yaml).toContain('crawl_mode: selected_roots');
expect(yaml).toContain('11111111-2222-3333-4444-555555555555');
expect(yaml).toContain('database-1');
expect(yaml).toContain('data-source-1');
expect(io.stdout()).toContain('Connection: notion-main');
expect(io.stdout()).toContain('No connections configured. Run `ktx setup` to add one.');
expect(io.stdout()).not.toContain('ktx connection add');
});
it('tests a configured connection through the native scan connector', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
await runKtxConnection(
{
command: 'add',
projectDir,
driver: 'sqlite',
connectionId: 'warehouse',
url: undefined,
schemas: [],
readonly: true,
force: false,
allowLiteralCredentials: false,
},
makeIo().io,
);
await writeConnections(projectDir, {
warehouse: { driver: 'sqlite', readonly: true },
});
const { connector, introspect, cleanup } = nativeConnector('sqlite', ['customers', 'orders']);
const createScanConnector = vi.fn(async () => connector);
const io = makeIo();
@ -602,22 +155,13 @@ describe('runKtxConnection', () => {
it('tests a configured Metabase connection through the Metabase runtime client', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
const projectConfig = parseKtxProjectConfig(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'));
await writeFile(
join(projectDir, 'ktx.yaml'),
serializeKtxProjectConfig({
...projectConfig,
connections: {
...projectConfig.connections,
prod_metabase: {
driver: 'metabase',
api_url: 'http://metabase.example.test',
api_key: 'mb_test',
},
},
}),
'utf-8',
);
await writeConnections(projectDir, {
prod_metabase: {
driver: 'metabase',
api_url: 'http://metabase.example.test',
api_key: 'mb_test',
},
});
const testConnection = vi.fn(async () => ({ success: true as const }));
const getDatabases = vi.fn(async () => [
{ id: 1, name: 'Analytics', engine: 'postgres', details: {}, is_sample: false },
@ -657,20 +201,9 @@ describe('runKtxConnection', () => {
it('cleans up the native scan connector when connection testing fails', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
await runKtxConnection(
{
command: 'add',
projectDir,
driver: 'sqlite',
connectionId: 'warehouse',
url: undefined,
schemas: [],
readonly: true,
force: false,
allowLiteralCredentials: false,
},
makeIo().io,
);
await writeConnections(projectDir, {
warehouse: { driver: 'sqlite', readonly: true },
});
const cleanup = vi.fn(async () => undefined);
const connector: KtxScanConnector = {
id: 'sqlite:warehouse',

335
packages/cli/src/connection.ts
View file

@ -1,108 +1,24 @@
import { cancel, confirm, isCancel } from '@clack/prompts';
import {
DEFAULT_METABASE_CLIENT_CONFIG,
DefaultMetabaseConnectionClientFactory,
type MetabaseRuntimeClient,
metabaseRuntimeConfigFromLocalConnection,
} from '@ktx/context/ingest';
import { type KtxLocalProject, loadKtxProject, serializeKtxProjectConfig } from '@ktx/context/project';
import { type KtxLocalProject, loadKtxProject } from '@ktx/context/project';
import type { KtxScanConnector } from '@ktx/context/scan';
import type { KtxConnectionMappingArgs } from './commands/connection-mapping.js';
import type { KtxCliIo } from './index.js';
import { createKtxCliScanConnector } from './local-scan-connectors.js';
import { profileMark } from './startup-profile.js';
profileMark('module:connection');
interface KtxNotionConnectionCliConfig {
authTokenRef: string;
crawlMode: 'all_accessible' | 'selected_roots';
rootPageIds: string[];
rootDatabaseIds: string[];
rootDataSourceIds: string[];
maxPagesPerRun?: number;
maxKnowledgeCreatesPerRun?: number;
maxKnowledgeUpdatesPerRun?: number;
}
type KtxConnectionInputMode = 'disabled';
export type KtxConnectionArgs =
| { command: 'list'; projectDir: string }
| {
command: 'add';
projectDir: string;
driver: string;
connectionId: string;
url?: string;
schemas: string[];
readonly: boolean;
force: boolean;
allowLiteralCredentials: boolean;
notion?: KtxNotionConnectionCliConfig;
}
| { command: 'test'; projectDir: string; connectionId: string }
| {
command: 'remove';
projectDir: string;
connectionId: string;
force: boolean;
inputMode?: KtxConnectionInputMode;
}
| {
command: 'map';
projectDir: string;
sourceConnectionId: string;
json: boolean;
};
interface KtxConnectionPromptAdapter {
confirm(options: { message: string; initialValue?: boolean }): Promise<boolean>;
cancel(message: string): void;
}
interface KtxConnectionIo extends KtxCliIo {
stdin?: { isTTY?: boolean };
}
| { command: 'test'; projectDir: string; connectionId: string };
interface KtxConnectionDeps {
createScanConnector?: typeof createKtxCliScanConnector;
createMetabaseClient?: typeof createDefaultMetabaseClient;
runMapping?: (argv: string[], io: KtxCliIo) => Promise<number>;
prompts?: KtxConnectionPromptAdapter;
}
function assertSafeConnectionId(connectionId: string): void {
if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(connectionId)) {
throw new Error(`Unsafe connection id: ${connectionId}`);
}
}
function isCredentialReference(value: string): boolean {
return value.startsWith('env:') || value.startsWith('file:');
}
function literalCredentialWarning(connectionId: string): string {
return `Warning: writing a literal credential URL to ktx.yaml for connection "${connectionId}". Prefer env:NAME or file:/path references.`;
}
function createClackConnectionPromptAdapter(): KtxConnectionPromptAdapter {
return {
async confirm(options: { message: string; initialValue?: boolean }): Promise<boolean> {
const value = await confirm(options);
return isCancel(value) ? false : value;
},
cancel(message: string): void {
cancel(message);
},
};
}
function isInteractiveConnectionIo(
args: Extract<KtxConnectionArgs, { command: 'remove' }>,
io: KtxConnectionIo,
): boolean {
return args.inputMode !== 'disabled' && io.stdin?.isTTY === true && io.stdout.isTTY === true;
}
async function cleanupConnector(connector: KtxScanConnector | null): Promise<void> {
@ -186,166 +102,17 @@ async function testMetabaseConnection(
}
}
interface BufferedIo extends KtxCliIo {
stdoutText(): string;
stderrText(): string;
}
function createBufferedIo(): BufferedIo {
let stdout = '';
let stderr = '';
return {
stdout: {
write(chunk: string) {
stdout += chunk;
},
},
stderr: {
write(chunk: string) {
stderr += chunk;
},
},
stdoutText() {
return stdout;
},
stderrText() {
return stderr;
},
};
}
function splitOutputLines(output: string): string[] {
return output
.split('\n')
.map((line) => line.trim())
.filter(Boolean);
}
async function runLowLevelMapping(
args: KtxConnectionMappingArgs,
argv: string[],
io: KtxCliIo,
deps: KtxConnectionDeps,
): Promise<number> {
if (deps.runMapping) {
return await deps.runMapping(argv, io);
}
const { runKtxConnectionMapping } = await import('./commands/connection-mapping.js');
return await runKtxConnectionMapping(args, io);
}
function parseMappingListJson(output: string): unknown[] {
const trimmed = output.trim();
if (!trimmed) {
return [];
}
const parsed = JSON.parse(trimmed) as unknown;
return Array.isArray(parsed) ? parsed : [];
}
async function runPublicConnectionMap(
args: Extract<KtxConnectionArgs, { command: 'map' }>,
io: KtxCliIo,
deps: KtxConnectionDeps,
): Promise<number> {
const refreshIo = createBufferedIo();
const refreshArgs: KtxConnectionMappingArgs = {
command: 'refresh',
projectDir: args.projectDir,
connectionId: args.sourceConnectionId,
autoAccept: true,
};
const refreshCode = await runLowLevelMapping(
refreshArgs,
['refresh', args.sourceConnectionId, '--auto-accept', '--project-dir', args.projectDir],
refreshIo,
deps,
);
if (refreshCode !== 0) {
io.stderr.write(
refreshIo.stderrText() ||
refreshIo.stdoutText() ||
`Failed to refresh mapping metadata for ${args.sourceConnectionId}\n`,
);
return refreshCode;
}
const validationIo = createBufferedIo();
const validationArgs: KtxConnectionMappingArgs = {
command: 'validate',
projectDir: args.projectDir,
connectionId: args.sourceConnectionId,
};
const validationCode = await runLowLevelMapping(
validationArgs,
['validate', args.sourceConnectionId, '--project-dir', args.projectDir],
validationIo,
deps,
);
if (validationCode !== 0) {
io.stderr.write(
validationIo.stderrText() || validationIo.stdoutText() || `Mapping validation failed for ${args.sourceConnectionId}\n`,
);
return validationCode;
}
const listIo = createBufferedIo();
const listArgv = ['list', args.sourceConnectionId, '--project-dir', args.projectDir];
const listArgs: KtxConnectionMappingArgs = {
command: 'list',
projectDir: args.projectDir,
connectionId: args.sourceConnectionId,
json: args.json,
};
const listCode = await runLowLevelMapping(listArgs, args.json ? [...listArgv, '--json'] : listArgv, listIo, deps);
if (listCode !== 0) {
io.stderr.write(listIo.stderrText() || listIo.stdoutText() || `Failed to list mappings for ${args.sourceConnectionId}\n`);
return listCode;
}
if (args.json) {
io.stdout.write(
`${JSON.stringify(
{
connectionId: args.sourceConnectionId,
refresh: { ok: true, output: splitOutputLines(refreshIo.stdoutText()) },
validation: { ok: true, output: splitOutputLines(validationIo.stdoutText()) },
mappings: parseMappingListJson(listIo.stdoutText()),
},
null,
2,
)}\n`,
);
return 0;
}
io.stdout.write(`Mapping: ${args.sourceConnectionId}\n`);
io.stdout.write(refreshIo.stdoutText());
io.stdout.write(validationIo.stdoutText());
io.stdout.write('\nMappings:\n');
io.stdout.write(listIo.stdoutText().trim() ? listIo.stdoutText() : 'No mappings found.\n');
io.stdout.write('\nNext:\n');
io.stdout.write(` ktx ingest ${args.sourceConnectionId}\n`);
io.stdout.write(` ktx dev mapping list ${args.sourceConnectionId}\n`);
return 0;
}
export async function runKtxConnection(
args: KtxConnectionArgs,
io: KtxConnectionIo = process,
io: KtxCliIo = process,
deps: KtxConnectionDeps = {},
): Promise<number> {
try {
if (args.command === 'map') {
return await runPublicConnectionMap(args, io, deps);
}
const project = await loadKtxProject({ projectDir: args.projectDir });
if (args.command === 'list') {
const entries = Object.entries(project.config.connections).sort(([a], [b]) => a.localeCompare(b));
if (entries.length === 0) {
io.stdout.write('No connections configured. Run `ktx connection add <id> --driver <driver>` to add one.\n');
io.stdout.write('No connections configured. Run `ktx setup` to add one.\n');
return 0;
}
const idWidth = Math.max('ID'.length, ...entries.map(([id]) => id.length));
@ -360,100 +127,6 @@ export async function runKtxConnection(
return 0;
}
if (args.command === 'add') {
assertSafeConnectionId(args.connectionId);
const hasLiteralCredentialUrl = !!args.url && !isCredentialReference(args.url);
if (hasLiteralCredentialUrl && !args.allowLiteralCredentials) {
throw new Error('Literal credential URLs require --allow-literal-credentials');
}
if (hasLiteralCredentialUrl) {
io.stderr.write(`${literalCredentialWarning(args.connectionId)}\n`);
}
if (project.config.connections[args.connectionId] && !args.force) {
throw new Error(`Connection "${args.connectionId}" already exists; pass --force to replace it`);
}
const connectionConfig =
args.driver === 'notion' && args.notion
? {
driver: 'notion',
auth_token_ref: args.notion.authTokenRef,
crawl_mode: args.notion.crawlMode,
root_page_ids: args.notion.rootPageIds,
root_database_ids: args.notion.rootDatabaseIds,
root_data_source_ids: args.notion.rootDataSourceIds,
...(args.notion.maxPagesPerRun !== undefined ? { max_pages_per_run: args.notion.maxPagesPerRun } : {}),
...(args.notion.maxKnowledgeCreatesPerRun !== undefined
? { max_knowledge_creates_per_run: args.notion.maxKnowledgeCreatesPerRun }
: {}),
...(args.notion.maxKnowledgeUpdatesPerRun !== undefined
? { max_knowledge_updates_per_run: args.notion.maxKnowledgeUpdatesPerRun }
: {}),
}
: {
driver: args.driver,
...(args.url ? { url: args.url } : {}),
...(args.schemas.length > 0 ? { schemas: args.schemas } : {}),
readonly: args.readonly,
};
const nextConfig = {
...project.config,
connections: {
...project.config.connections,
[args.connectionId]: connectionConfig,
},
};
await project.fileStore.writeFile(
'ktx.yaml',
serializeKtxProjectConfig(nextConfig),
'ktx',
'ktx@example.com',
`Update KTX connection: ${args.connectionId}`,
);
io.stdout.write(`Connection: ${args.connectionId}\n`);
io.stdout.write(`Driver: ${args.driver}\n`);
return 0;
}
if (args.command === 'remove') {
if (!project.config.connections[args.connectionId]) {
throw new Error(`Connection "${args.connectionId}" is not configured in ktx.yaml`);
}
if (!args.force) {
if (!isInteractiveConnectionIo(args, io)) {
throw new Error(
`connection remove ${args.connectionId} requires --force when input is disabled or not interactive`,
);
}
const prompts = deps.prompts ?? createClackConnectionPromptAdapter();
const confirmed = await prompts.confirm({
message: `Remove connection "${args.connectionId}" from ktx.yaml? Ingested artifacts will remain in .ktx/.`,
initialValue: false,
});
if (!confirmed) {
prompts.cancel('Connection removal cancelled.');
return 1;
}
}
const { [args.connectionId]: _removedConnection, ...connections } = project.config.connections;
const nextConfig = {
...project.config,
connections,
};
await project.fileStore.writeFile(
'ktx.yaml',
serializeKtxProjectConfig(nextConfig),
'ktx',
'ktx@example.com',
`Remove KTX connection: ${args.connectionId}`,
);
io.stdout.write('Connection removed from ktx.yaml.\n');
io.stdout.write('Ingested artifacts from this connection remain in .ktx/. Run ktx dev artifacts to inspect.\n');
return 0;
}
if (normalizedConnectionDriver(project, args.connectionId) === 'metabase') {
const result = await testMetabaseConnection(
project,

107
packages/cli/src/context-build-view.test.ts
View file

@ -168,6 +168,15 @@ describe('renderContextBuildView', () => {
expect(output).toContain('(0/1 · 1m05s)');
});
it('renders project directory when provided', () => {
const state = initViewState([
{ connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] },
]);
const output = renderContextBuildView(state, { styled: false, projectDir: '/tmp/project' });
expect(output).toContain('Project: /tmp/project');
});
it('renders dynamic separator matching header width', () => {
const state = initViewState([
{ connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] },
@ -222,6 +231,38 @@ describe('renderContextBuildView', () => {
expect(output).toContain('(15s)');
});
it('shows how long a running target has gone without a progress update', () => {
const state = initViewState([
{ connectionId: 'notion-main', driver: 'notion', operation: 'source-ingest', debugCommand: '', steps: ['source-ingest', 'memory-update'] },
]);
state.contextSources[0].status = 'running';
state.contextSources[0].startedAt = 1_000;
state.contextSources[0].elapsedMs = 113_000;
state.contextSources[0].progressUpdatedAtMs = 46_000;
state.contextSources[0].detailLine = '[45%] No work units to process; finalizing ingest';
const output = renderContextBuildView(state, { styled: false });
expect(output).toContain('No work units to process; finalizing ingest');
expect(output).toContain('last update 1m08s ago');
expect(output).toContain('(1m53s)');
});
it('does not show progress age while updates are recent', () => {
const state = initViewState([
{ connectionId: 'notion-main', driver: 'notion', operation: 'source-ingest', debugCommand: '', steps: ['source-ingest', 'memory-update'] },
]);
state.contextSources[0].status = 'running';
state.contextSources[0].startedAt = 1_000;
state.contextSources[0].elapsedMs = 40_000;
state.contextSources[0].progressUpdatedAtMs = 25_000;
state.contextSources[0].detailLine = '[45%] Planning work units';
const output = renderContextBuildView(state, { styled: false });
expect(output).not.toContain('last update');
});
it('renders completion summary when all targets are done', () => {
const state = initViewState([
{ connectionId: 'warehouse', driver: 'postgres', operation: 'scan', debugCommand: '', steps: ['scan'] },
@ -494,6 +535,7 @@ describe('runContextBuild', () => {
const output = io.stdout();
expect(output).toContain('Building KTX context');
expect(output).toContain('Project: /tmp/project');
expect(output).toContain('Primary sources:');
expect(output).toContain('warehouse');
expect(output).toContain('Context sources:');
@ -516,7 +558,10 @@ describe('runContextBuild', () => {
expect.objectContaining({ connectionId: 'warehouse', operation: 'scan' }),
expect.objectContaining({ scanMode: 'enriched', detectRelationships: true }),
expect.anything(),
{},
expect.objectContaining({
scanProgress: expect.anything(),
ingestProgress: expect.any(Function),
}),
);
});
@ -632,6 +677,43 @@ describe('runContextBuild', () => {
]);
});
it('publishes structured target progress without expanding the compact source rows', async () => {
const io = makeIo({ isTTY: true });
const project = projectWithConnections({
warehouse: { driver: 'postgres' },
});
const progressUpdates: Array<Array<{ connectionId: string; percent?: number; message?: string }>> = [];
const executeTarget = vi.fn(async (target, _args, _targetIo, deps) => {
await deps.scanProgress?.update(0.37, 'Generating descriptions 3/8 tables', { transient: true });
return successResult(target.connectionId, target.driver, target.operation);
});
await runContextBuild(
project,
{ projectDir: '/tmp/project', inputMode: 'disabled' },
io.io,
{
executeTarget,
now: () => 1000,
onSourceProgress: (sources) => {
progressUpdates.push(
sources.map((s) => ({
connectionId: s.connectionId,
...(s.percent !== undefined ? { percent: s.percent } : {}),
...(s.message !== undefined ? { message: s.message } : {}),
})),
);
},
sourceProgressThrottleMs: 0,
},
);
expect(progressUpdates).toContainEqual([
{ connectionId: 'warehouse', percent: 37, message: 'Generating descriptions 3/8 tables' },
]);
expect(io.stdout()).toContain('Generating descriptions 3/8 tables');
});
it('returns report IDs and artifact paths parsed from target output', async () => {
const io = makeIo();
const project = projectWithConnections({
@ -748,4 +830,27 @@ describe('viewStateFromSourceProgress', () => {
expect(output).toContain('dbt-main');
expect(output).toContain('ingesting...');
});
it('renders persisted percent and message as compact source-row progress', () => {
const state = viewStateFromSourceProgress(
[
{
connectionId: 'warehouse',
operation: 'scan',
status: 'running',
startedAtMs: 900,
percent: 63,
message: 'Building embeddings 2/4 batches',
updatedAtMs: 950,
},
],
1000,
);
const output = renderContextBuildView(state, { styled: false });
expect(output).toContain('warehouse');
expect(output).toContain('63%');
expect(output).toContain('Building embeddings 2/4 batches');
expect(output.match(/warehouse/g)).toHaveLength(1);
});
});

139
packages/cli/src/context-build-view.ts
View file

@ -1,9 +1,12 @@
import { spawn } from 'node:child_process';
import { mkdirSync, openSync } from 'node:fs';
import { join, resolve } from 'node:path';
import type { KtxProgressPort, KtxProgressUpdateOptions } from '@ktx/context/scan';
import type { KtxCliIo } from './index.js';
import type { KtxIngestProgressUpdate } from './ingest.js';
import type {
KtxPublicIngestArgs,
KtxPublicIngestDeps,
KtxPublicIngestPlanTarget,
KtxPublicIngestProject,
KtxPublicIngestTargetResult,
@ -25,6 +28,7 @@ export interface ContextBuildTargetState {
failureText: string | null;
startedAt: number | null;
elapsedMs: number;
progressUpdatedAtMs: number | null;
}
export interface ContextBuildViewState {
@ -58,6 +62,9 @@ export interface ContextBuildSourceProgressUpdate {
status: 'queued' | 'running' | 'done' | 'failed';
startedAtMs?: number;
elapsedMs?: number;
percent?: number;
message?: string;
updatedAtMs?: number;
summaryText?: string;
}
@ -67,6 +74,7 @@ export interface ContextBuildDeps {
setupKeystroke?: (onDetach: () => void, onCtrlC: () => void) => (() => void) | null;
onDetach?: () => void;
onSourceProgress?: (sources: ContextBuildSourceProgressUpdate[]) => void;
sourceProgressThrottleMs?: number;
}
// --- Rendering ---
@ -121,6 +129,7 @@ function extractPercent(detailLine: string | null): number | null {
const BAR_WIDTH = 12;
const BAR_FILLED = '█';
const BAR_EMPTY = '░';
const STALE_PROGRESS_UPDATE_MS = 30_000;
function renderProgressBar(percent: number, styled: boolean): string {
const filled = Math.round((percent / 100) * BAR_WIDTH);
@ -129,6 +138,19 @@ function renderProgressBar(percent: number, styled: boolean): string {
return styled ? cyan(bar) : bar;
}
function staleProgressText(target: ContextBuildTargetState, styled: boolean): string | null {
if (target.startedAt === null || target.progressUpdatedAtMs === null || target.elapsedMs <= 0) {
return null;
}
const currentTimeMs = target.startedAt + target.elapsedMs;
const staleMs = currentTimeMs - target.progressUpdatedAtMs;
if (staleMs < STALE_PROGRESS_UPDATE_MS) {
return null;
}
const text = `last update ${formatDuration(staleMs)} ago`;
return styled ? dim(text) : text;
}
function targetDetail(target: ContextBuildTargetState, styled: boolean): string {
if (target.status === 'done') {
const parts: string[] = [];
@ -150,6 +172,8 @@ function targetDetail(target: ContextBuildTargetState, styled: boolean): string
parts.push(`${renderProgressBar(percent, styled)} ${percent}%`);
}
parts.push(progressText);
const stale = staleProgressText(target, styled);
if (stale) parts.push(stale);
if (elapsed) parts.push(styled ? dim(elapsed) : elapsed);
return parts.join(' ');
}
@ -207,6 +231,7 @@ export function renderContextBuildView(
'',
header,
separator,
...(options.projectDir ? [` Project: ${options.projectDir}`] : []),
...renderTargetGroup('Primary sources', state.primarySources, state.frame, styled, width),
...renderTargetGroup('Context sources', state.contextSources, state.frame, styled, width),
'',
@ -312,15 +337,42 @@ function createCaptureIo(onProgress: (message: string) => void, isTTY: boolean):
// --- Source progress helpers ---
function progressFieldsFromDetailLine(
detailLine: string | null,
updatedAtMs: number | null,
): Pick<ContextBuildSourceProgressUpdate, 'percent' | 'message' | 'updatedAtMs'> {
if (!detailLine) return {};
const percent = extractPercent(detailLine);
const message = detailLine.replace(/^\[\d+%\]\s*/, '');
return {
...(percent !== null ? { percent } : {}),
...(message ? { message } : {}),
...(updatedAtMs !== null ? { updatedAtMs } : {}),
};
}
function detailLineFromProgressSource(source: ContextBuildSourceProgressUpdate): string | null {
if (!source.message) return null;
if (typeof source.percent === 'number' && Number.isFinite(source.percent)) {
const percent = Math.max(0, Math.min(100, Math.round(source.percent)));
return `[${percent}%] ${source.message}`;
}
return source.message;
}
function collectSourceProgress(targets: ContextBuildTargetState[]): ContextBuildSourceProgressUpdate[] {
return targets.map((t) => ({
connectionId: t.target.connectionId,
operation: t.target.operation,
status: t.status,
...(t.startedAt !== null ? { startedAtMs: t.startedAt } : {}),
...(t.elapsedMs > 0 ? { elapsedMs: t.elapsedMs } : {}),
...(t.summaryText ? { summaryText: t.summaryText } : {}),
}));
return targets.map((t) => {
const progressFields = progressFieldsFromDetailLine(t.detailLine, t.progressUpdatedAtMs);
return {
connectionId: t.target.connectionId,
operation: t.target.operation,
status: t.status,
...(t.startedAt !== null ? { startedAtMs: t.startedAt } : {}),
...(t.elapsedMs > 0 ? { elapsedMs: t.elapsedMs } : {}),
...progressFields,
...(t.summaryText ? { summaryText: t.summaryText } : {}),
};
});
}
export function viewStateFromSourceProgress(
@ -331,11 +383,12 @@ export function viewStateFromSourceProgress(
const makeTarget = (s: ContextBuildSourceProgressUpdate): ContextBuildTargetState => ({
target: { connectionId: s.connectionId, driver: '', operation: s.operation, debugCommand: '', steps: [] },
status: s.status,
detailLine: null,
detailLine: detailLineFromProgressSource(s),
summaryText: s.summaryText ?? null,
failureText: null,
startedAt: s.startedAtMs ?? null,
elapsedMs: s.status === 'running' && s.startedAtMs ? now - s.startedAtMs : (s.elapsedMs ?? 0),
progressUpdatedAtMs: s.updatedAtMs ?? null,
});
return {
@ -467,6 +520,7 @@ function makeTargetState(target: KtxPublicIngestPlanTarget): ContextBuildTargetS
failureText: null,
startedAt: null,
elapsedMs: 0,
progressUpdatedAtMs: null,
};
}
@ -571,6 +625,34 @@ export function initViewState(
};
}
function formatProgressDetail(update: Pick<KtxIngestProgressUpdate, 'percent' | 'message'>): string {
const percent = Math.max(0, Math.min(100, Math.round(update.percent)));
return `[${percent}%] ${update.message}`;
}
function createContextBuildProgressPort(
onProgress: (update: KtxIngestProgressUpdate) => void,
state: { progress: number } = { progress: 0 },
start = 0,
weight = 1,
): KtxProgressPort {
return {
async update(value: number, message?: string, options?: KtxProgressUpdateOptions): Promise<void> {
const absoluteValue = start + Math.max(0, Math.min(1, value)) * weight;
state.progress = Math.max(state.progress, Math.min(1, absoluteValue));
if (!message) return;
onProgress({
percent: Math.max(0, Math.min(100, Math.round(state.progress * 100))),
message,
...(options?.transient !== undefined ? { transient: options.transient } : {}),
});
},
startPhase(phaseWeight: number): KtxProgressPort {
return createContextBuildProgressPort(onProgress, state, state.progress, weight * phaseWeight);
},
};
}
export async function runContextBuild(
project: KtxPublicIngestProject,
args: ContextBuildArgs,
@ -614,6 +696,19 @@ export async function runContextBuild(
const execTarget = deps.executeTarget ?? executePublicIngestTarget;
const reportIds = new Set<string>();
const artifactPaths = new Set<string>();
const sourceProgressThrottleMs = deps.sourceProgressThrottleMs ?? 750;
let lastSourceProgressPublishedAt = Number.NEGATIVE_INFINITY;
const publishSourceProgress = (force = false): boolean => {
if (!deps.onSourceProgress) return false;
const now = nowFn();
if (!force && now - lastSourceProgressPublishedAt < sourceProgressThrottleMs) {
return false;
}
lastSourceProgressPublishedAt = now;
deps.onSourceProgress(collectSourceProgress(orderedTargets));
return true;
};
let detached = false;
let exiting = false;
@ -666,20 +761,34 @@ export async function runContextBuild(
targetState.status = 'running';
targetState.startedAt = nowFn();
paint(true);
deps.onSourceProgress?.(collectSourceProgress(orderedTargets));
publishSourceProgress(true);
let hasPendingProgressPublish = false;
const updateTargetProgress = (update: KtxIngestProgressUpdate) => {
targetState.detailLine = formatProgressDetail(update);
targetState.progressUpdatedAtMs = nowFn();
paint(true);
hasPendingProgressPublish = !publishSourceProgress(false);
};
const capture = createCaptureIo(
(message) => {
targetState.detailLine = message;
targetState.progressUpdatedAtMs = nowFn();
paint(true);
hasPendingProgressPublish = !publishSourceProgress(false);
},
false,
);
const progressDeps: KtxPublicIngestDeps = {
scanProgress: createContextBuildProgressPort(updateTargetProgress),
ingestProgress: updateTargetProgress,
};
let result: KtxPublicIngestTargetResult | null = null;
let thrownError: unknown = null;
try {
result = await execTarget(targetState.target, runArgs, capture.io, {});
result = await execTarget(targetState.target, runArgs, capture.io, progressDeps);
} catch (error) {
if (exiting) {
throw error;
@ -687,6 +796,10 @@ export async function runContextBuild(
thrownError = error;
}
if (hasPendingProgressPublish) {
publishSourceProgress(true);
}
targetState.elapsedMs = nowFn() - (targetState.startedAt ?? nowFn());
const failed = thrownError !== null || result?.steps.some((s) => s.status === 'failed') === true;
targetState.status = failed ? 'failed' : 'done';
@ -712,7 +825,7 @@ export async function runContextBuild(
if (failed) hasFailure = true;
paint(true);
deps.onSourceProgress?.(collectSourceProgress(orderedTargets));
publishSourceProgress(true);
}
} finally {
if (spinnerInterval) clearInterval(spinnerInterval);
@ -728,7 +841,7 @@ export async function runContextBuild(
}
if (!repainter) {
io.stdout.write(renderContextBuildView(state, { styled: false }));
io.stdout.write(renderContextBuildView(state, { ...viewOpts, styled: false }));
} else {
paint(false);
}

6
packages/cli/src/demo-assets.test.ts
View file

@ -95,7 +95,7 @@ describe('demo assets', () => {
await expect(access(packagedDemoAssetPath('semantic-layer/dbt-main/mart_arr_daily.yaml'))).resolves.toBeUndefined();
await expect(access(packagedDemoAssetPath('semantic-layer/postgres-warehouse/mart_account_activity.yaml'))).resolves.toBeUndefined();
await expect(access(packagedDemoAssetPath('knowledge/global/orbit-company-overview.md'))).resolves.toBeUndefined();
await expect(access(packagedDemoAssetPath('wiki/global/orbit-company-overview.md'))).resolves.toBeUndefined();
await expect(access(packagedDemoAssetPath('links/provenance.json'))).resolves.toBeUndefined();
await expect(access(packagedDemoAssetPath('reports/seeded-demo-report.json'))).resolves.toBeUndefined();
});
@ -108,7 +108,7 @@ describe('demo assets', () => {
await expect(access(join(projectDir, 'state.sqlite'))).resolves.toBeUndefined();
await expect(access(join(projectDir, 'reports'))).resolves.toBeUndefined();
await expect(access(join(projectDir, 'semantic-layer'))).resolves.toBeUndefined();
await expect(access(join(projectDir, 'knowledge'))).resolves.toBeUndefined();
await expect(access(join(projectDir, 'wiki'))).resolves.toBeUndefined();
await expect(access(join(projectDir, 'replays', 'replay.memory-flow.v1.json'))).resolves.toBeUndefined();
await expect(access(join(projectDir, 'raw-sources'))).resolves.toBeUndefined();
await expect(access(join(projectDir, '_schema'))).rejects.toMatchObject({ code: 'ENOENT' });
@ -129,7 +129,7 @@ describe('demo assets', () => {
await ensureSeededDemoProject({ projectDir, force: false });
await expect(access(join(projectDir, 'semantic-layer', 'dbt-main', 'mart_arr_daily.yaml'))).resolves.toBeUndefined();
await expect(access(join(projectDir, 'knowledge', 'global', 'orbit-company-overview.md'))).resolves.toBeUndefined();
await expect(access(join(projectDir, 'wiki', 'global', 'orbit-company-overview.md'))).resolves.toBeUndefined();
await expect(access(join(projectDir, 'links', 'provenance.json'))).resolves.toBeUndefined();
await expect(access(join(projectDir, 'reports', 'seeded-demo-report.json'))).resolves.toBeUndefined();
});

6
packages/cli/src/demo-assets.ts
View file

@ -29,7 +29,7 @@ const REQUIRED_SEEDED_ASSET_PATHS = [
DEMO_REPLAY_FILE,
join('semantic-layer', 'dbt-main', 'mart_arr_daily.yaml'),
join('semantic-layer', 'postgres-warehouse', 'mart_account_activity.yaml'),
join('knowledge', 'global', 'orbit-company-overview.md'),
join('wiki', 'global', 'orbit-company-overview.md'),
] as const;
function assetDir(): string {
@ -131,7 +131,7 @@ export async function ensureDemoProject(options: EnsureDemoProjectOptions): Prom
}
await mkdir(projectDir, { recursive: true });
for (const relativeDir of ['reports', 'semantic-layer', 'knowledge', 'replays', 'raw-sources', 'links']) {
for (const relativeDir of ['reports', 'semantic-layer', 'wiki', 'replays', 'raw-sources', 'links']) {
await mkdir(join(projectDir, relativeDir), { recursive: true });
}
@ -157,7 +157,7 @@ async function copySeededAssetDirectories(projectDir: string): Promise<void> {
await Promise.all([
copyDirIfExists(join(src, 'semantic-layer'), join(dest, 'semantic-layer')),
copyDirIfExists(join(src, 'knowledge'), join(dest, 'knowledge')),
copyDirIfExists(join(src, 'wiki'), join(dest, 'wiki')),
copyDirIfExists(join(src, 'raw-sources'), join(dest, 'raw-sources')),
copyDirIfExists(join(src, 'links'), join(dest, 'links')),
copyDirIfExists(join(src, 'reports'), join(dest, 'reports')),

455
packages/cli/src/dev.test.ts
View file

@ -29,11 +29,14 @@ describe('dev Commander tree', () => {
await expect(runKtxCli(['dev', '--help'], testIo.io)).resolves.toBe(0);
expect(testIo.stdout()).toContain('Usage: ktx dev [options] [command]');
for (const command of ['init', 'runtime', 'scan', 'ingest', 'mapping']) {
for (const command of ['init', 'runtime']) {
expect(testIo.stdout()).toContain(command);
}
for (const removed of [
'doctor',
'scan',
'ingest',
'mapping',
'knowledge',
'model',
'replay',
@ -102,6 +105,13 @@ describe('dev Commander tree', () => {
it('rejects removed dev command groups', async () => {
for (const argv of [
['dev', 'doctor', 'setup'],
['dev', 'runtime', 'doctor'],
['dev', 'runtime', 'prune', '--dry-run'],
['dev', 'scan', 'warehouse'],
['dev', 'ingest', 'run'],
['dev', 'mapping', 'list'],
['dev', 'completion', 'zsh'],
['dev', '__complete', '--shell', 'zsh', '--position', '2', '--', 'ktx', ''],
['dev', 'knowledge', 'list'],
['dev', 'model', 'list'],
['dev', 'artifacts'],
@ -117,90 +127,15 @@ describe('dev Commander tree', () => {
it.each([
{
argv: ['dev', 'runtime', '--help'],
expected: ['Usage: ktx dev runtime', 'install', 'start', 'stop', 'status', 'doctor', 'prune'],
expected: ['Usage: ktx dev runtime', 'install', 'start', 'stop', 'status'],
},
{
argv: ['dev', 'scan', '--help'],
expected: [
'Usage: ktx dev scan',
'--mode <mode>',
'structural',
'relationships',
'--dry-run',
'status',
'report',
'relationships',
'relationship-apply',
'relationship-feedback',
'relationship-calibration',
'relationship-thresholds',
],
argv: ['scan', '--help'],
expected: ['Usage: ktx scan [options] <connectionId>', '--mode <mode>', 'structural', 'relationships', '--dry-run'],
},
{
argv: ['dev', 'scan', 'report', '--help'],
expected: ['Usage: ktx dev scan report [options] <runId>', '<runId>', '--json'],
},
{
argv: ['dev', 'scan', 'relationships', '--help'],
expected: [
'Usage: ktx dev scan relationships [options] <runId>',
'--status <status>',
'--limit <count>',
'--accept <candidateId>',
'--reject <candidateId>',
'--note <text>',
'--reviewer <name>',
'--json',
],
},
{
argv: ['dev', 'scan', 'relationship-apply', '--help'],
expected: [
'Usage: ktx dev scan relationship-apply [options] <runId>',
'--all-accepted',
'--candidate <candidateId>',
'--dry-run',
],
},
{
argv: ['dev', 'scan', 'relationship-thresholds', '--help'],
expected: [
'Usage: ktx dev scan relationship-thresholds [options]',
'--connection <connectionId>',
'--min-total-labels <count>',
'--min-accepted-labels <count>',
'--min-rejected-labels <count>',
'--json',
],
},
{
argv: ['dev', 'scan', 'relationship-feedback', '--help'],
expected: [
'Usage: ktx dev scan relationship-feedback [options]',
'--connection <connectionId>',
'--decision <decision>',
'--json',
'--jsonl',
],
},
{
argv: ['dev', 'scan', 'relationship-calibration', '--help'],
expected: [
'Usage: ktx dev scan relationship-calibration [options]',
'--connection <connectionId>',
'--decision <decision>',
'--accept-threshold <value>',
'--review-threshold <value>',
'--json',
],
},
{
argv: ['dev', 'ingest', 'run', '--help'],
expected: ['Usage: ktx dev ingest run [options]', '--connection-id <connectionId>', '--adapter <adapter>'],
},
{
argv: ['dev', 'mapping', 'sync-state', 'set', '--help'],
expected: ['Usage: ktx dev mapping sync-state set [options] <connectionId>', '--mode <mode>'],
argv: ['ingest', 'run', '--help'],
expected: ['Usage: ktx ingest run [options]', '--connection-id <connectionId>', '--adapter <adapter>'],
},
])('prints generated nested help for $argv', async ({ argv, expected }) => {
const io = makeIo();
@ -213,18 +148,22 @@ describe('dev Commander tree', () => {
for (const text of expected) {
expect(io.stdout()).toContain(text);
}
if (argv.join(' ') === 'dev runtime --help') {
expect(io.stdout()).not.toContain('prune');
expect(io.stdout()).not.toContain('doctor');
}
expect(io.stderr()).toBe('');
expect(doctor).not.toHaveBeenCalled();
expect(ingest).not.toHaveBeenCalled();
expect(scan).not.toHaveBeenCalled();
});
it('dispatches dev scan through Commander with injected dependencies', async () => {
it('dispatches top-level scan through Commander with injected dependencies', async () => {
const scanIo = makeIo();
const scan = vi.fn(async () => 0);
await expect(
runKtxCli(['dev', 'scan', 'warehouse', '--project-dir', '/tmp/project', '--dry-run'], scanIo.io, { scan }),
runKtxCli(['scan', 'warehouse', '--project-dir', '/tmp/project', '--dry-run'], scanIo.io, { scan }),
).resolves.toBe(0);
expect(scan).toHaveBeenCalledWith(
@ -244,12 +183,12 @@ describe('dev Commander tree', () => {
expect(scanIo.stderr()).toBe('Project: /tmp/project\n');
});
it('dispatches dev scan --mode relationships through Commander', async () => {
it('dispatches top-level scan --mode relationships through Commander', async () => {
const io = makeIo();
const scan = vi.fn(async () => 0);
await expect(
runKtxCli(['dev', 'scan', 'warehouse', '--project-dir', '/tmp/project', '--mode', 'relationships'], io.io, {
runKtxCli(['scan', 'warehouse', '--project-dir', '/tmp/project', '--mode', 'relationships'], io.io, {
scan,
}),
).resolves.toBe(0);
@ -275,375 +214,53 @@ describe('dev Commander tree', () => {
const io = makeIo();
const scan = vi.fn(async () => 0);
await expect(runKtxCli(['dev', 'scan', 'warehouse', option], io.io, { scan })).resolves.toBe(1);
await expect(runKtxCli(['scan', 'warehouse', option], io.io, { scan })).resolves.toBe(1);
expect(scan).not.toHaveBeenCalled();
expect(io.stderr()).toContain(`unknown option '${option}'`);
});
it('rejects dev scan without a connection id or subcommand', async () => {
it('rejects scan without a connection id', async () => {
const io = makeIo();
const scan = vi.fn(async () => 0);
await expect(runKtxCli(['dev', 'scan', '--dry-run'], io.io, { scan })).resolves.toBe(1);
await expect(runKtxCli(['scan', '--dry-run'], io.io, { scan })).resolves.toBe(1);
expect(scan).not.toHaveBeenCalled();
expect(io.stdout()).toContain('Usage: ktx dev scan');
expect(io.stderr()).toContain('ktx dev scan requires <connectionId> or a subcommand');
expect(io.stderr()).toMatch(/missing required argument/i);
});
it('rejects invalid scan modes before dispatch', async () => {
const io = makeIo();
const scan = vi.fn(async () => 0);
await expect(runKtxCli(['dev', 'scan', 'warehouse', '--mode', 'deep'], io.io, { scan })).resolves.toBe(1);
await expect(runKtxCli(['scan', 'warehouse', '--mode', 'deep'], io.io, { scan })).resolves.toBe(1);
expect(scan).not.toHaveBeenCalled();
expect(io.stderr()).toContain("argument 'deep' is invalid");
expect(io.stderr()).toContain('Allowed choices are structural, enriched, relationships');
});
it('prints dev scan subcommand help with the canonical command name', async () => {
it.each([
['scan', 'report', 'scan-run-1'],
['scan', 'relationships', 'scan-run-1'],
])('rejects removed scan subcommand %s %s', async (command, subcommand, runId) => {
const io = makeIo();
const scan = vi.fn(async () => 0);
await expect(runKtxCli(['dev', 'scan', 'report', '--help'], io.io, { scan })).resolves.toBe(0);
expect(io.stdout()).toContain('--project-dir is inherited from `ktx dev scan`');
expect(io.stdout()).not.toContain('--project-dir is inherited from `ktx scan`');
expect(scan).not.toHaveBeenCalled();
});
it('dispatches dev scan report in human and json modes', async () => {
const humanIo = makeIo();
const jsonIo = makeIo();
const scan = vi.fn(async () => 0);
await expect(
runKtxCli(['dev', 'scan', 'report', 'scan-run-1', '--project-dir', '/tmp/project'], humanIo.io, { scan }),
).resolves.toBe(0);
await expect(
runKtxCli(['dev', 'scan', 'report', 'scan-run-2', '--project-dir', '/tmp/project', '--json'], jsonIo.io, {
scan,
}),
).resolves.toBe(0);
expect(scan).toHaveBeenNthCalledWith(
1,
{ command: 'report', projectDir: '/tmp/project', runId: 'scan-run-1', json: false },
humanIo.io,
);
expect(scan).toHaveBeenNthCalledWith(
2,
{ command: 'report', projectDir: '/tmp/project', runId: 'scan-run-2', json: true },
jsonIo.io,
);
});
it('dispatches dev scan relationships with filters through Commander', async () => {
const io = makeIo();
const scan = vi.fn(async () => 0);
await expect(
runKtxCli(
[
'dev',
'scan',
'relationships',
'scan-run-review',
'--project-dir',
'/tmp/project',
'--status',
'rejected',
'--limit',
'5',
'--json',
],
io.io,
{ scan },
),
).resolves.toBe(0);
expect(scan).toHaveBeenCalledWith(
{
command: 'relationships',
projectDir: '/tmp/project',
runId: 'scan-run-review',
status: 'rejected',
json: true,
limit: 5,
},
io.io,
);
expect(io.stderr()).toBe('');
});
it('dispatches dev scan relationship decision recording through Commander', async () => {
const io = makeIo();
const scan = vi.fn(async () => 0);
await expect(
runKtxCli(
[
'dev',
'scan',
'relationships',
'scan-run-review',
'--project-dir',
'/tmp/project',
'--accept',
'orders:orders.customer_id->customers:customers.id',
'--reviewer',
'Andrey',
'--note',
'Looks right',
'--json',
],
io.io,
{ scan },
),
).resolves.toBe(0);
expect(scan).toHaveBeenCalledWith(
{
command: 'relationshipDecision',
projectDir: '/tmp/project',
runId: 'scan-run-review',
candidateId: 'orders:orders.customer_id->customers:customers.id',
decision: 'accepted',
reviewer: 'Andrey',
note: 'Looks right',
json: true,
},
io.io,
);
expect(io.stderr()).toBe('');
});
it.each(['--accept', '--reject'])('rejects empty relationship decision candidate ids for %s', async (option) => {
const io = makeIo();
const scan = vi.fn(async () => 0);
await expect(
runKtxCli(['dev', 'scan', 'relationships', 'scan-run-review', option, ''], io.io, { scan }),
).resolves.toBe(1);
await expect(runKtxCli([command, subcommand, runId], io.io, { scan })).resolves.toBe(1);
expect(scan).not.toHaveBeenCalled();
expect(io.stderr()).toContain('must not be empty');
expect(io.stderr()).toMatch(/too many arguments|unknown command|error:/);
});
it('rejects relationship feedback JSON and JSONL output together', async () => {
const io = makeIo();
const scan = vi.fn(async () => 0);
await expect(
runKtxCli(['dev', 'scan', 'relationship-feedback', '--json', '--jsonl'], io.io, { scan }),
).resolves.toBe(1);
expect(scan).not.toHaveBeenCalled();
expect(io.stderr()).toMatch(/conflict|cannot be used/i);
});
it('dispatches relationship apply command args', async () => {
const io = makeIo();
const scan = vi.fn(async () => 0);
await expect(
runKtxCli(
[
'dev',
'scan',
'relationship-apply',
'scan-run-a',
'--project-dir',
'/tmp/project',
'--candidate',
'orders:orders.customer_id->customers:customers.id',
'--dry-run',
'--json',
],
io.io,
{ scan },
),
).resolves.toBe(0);
expect(scan).toHaveBeenCalledWith(
{
command: 'relationshipApply',
projectDir: '/tmp/project',
runId: 'scan-run-a',
applyAllAccepted: false,
candidateIds: ['orders:orders.customer_id->customers:customers.id'],
dryRun: true,
json: true,
},
io.io,
);
});
it('dispatches scan relationship feedback command with filters and JSONL output', async () => {
const io = makeIo();
const scan = vi.fn(async () => 0);
await expect(
runKtxCli(
[
'dev',
'scan',
'relationship-feedback',
'--project-dir',
'/tmp/project',
'--connection',
'warehouse',
'--decision',
'accepted',
'--jsonl',
],
io.io,
{ scan },
),
).resolves.toBe(0);
expect(scan).toHaveBeenCalledWith(
{
command: 'relationshipFeedback',
projectDir: '/tmp/project',
connectionId: 'warehouse',
decision: 'accepted',
json: false,
jsonl: true,
},
io.io,
);
});
it('dispatches scan relationship calibration command with thresholds', async () => {
const io = makeIo();
const scan = vi.fn(async () => 0);
await expect(
runKtxCli(
[
'dev',
'scan',
'relationship-calibration',
'--project-dir',
'/tmp/project',
'--connection',
'warehouse',
'--decision',
'rejected',
'--accept-threshold',
'0.9',
'--review-threshold',
'0.5',
'--json',
],
io.io,
{ scan },
),
).resolves.toBe(0);
expect(scan).toHaveBeenCalledWith(
{
command: 'relationshipCalibration',
projectDir: '/tmp/project',
connectionId: 'warehouse',
decision: 'rejected',
acceptThreshold: 0.9,
reviewThreshold: 0.5,
json: true,
},
io.io,
);
});
it('dispatches relationship threshold advice command args', async () => {
const io = makeIo();
const scan = vi.fn(async () => 0);
await expect(
runKtxCli(
[
'dev',
'scan',
'relationship-thresholds',
'--project-dir',
'/tmp/project',
'--connection',
'warehouse',
'--min-total-labels',
'12',
'--min-accepted-labels',
'4',
'--min-rejected-labels',
'3',
'--json',
],
io.io,
{ scan },
),
).resolves.toBe(0);
expect(scan).toHaveBeenCalledWith(
{
command: 'relationshipThresholds',
projectDir: '/tmp/project',
connectionId: 'warehouse',
minTotalLabels: 12,
minAcceptedLabels: 4,
minRejectedLabels: 3,
json: true,
},
io.io,
);
});
it('rejects invalid relationship calibration thresholds before dispatch', async () => {
const io = makeIo();
const scan = vi.fn(async () => 0);
await expect(
runKtxCli(['dev', 'scan', 'relationship-calibration', '--accept-threshold', '1.5'], io.io, { scan }),
).resolves.toBe(1);
expect(scan).not.toHaveBeenCalled();
expect(io.stderr()).toContain('Allowed range is 0 through 1');
});
it('rejects relationship accept and reject options together before dispatch', async () => {
const io = makeIo();
const scan = vi.fn(async () => 0);
await expect(
runKtxCli(
[
'dev',
'scan',
'relationships',
'scan-run-review',
'--accept',
'orders:orders.customer_id->customers:customers.id',
'--reject',
'orders:orders.customer_id->customers:customers.id',
],
io.io,
{ scan },
),
).resolves.toBe(1);
expect(scan).not.toHaveBeenCalled();
expect(io.stderr()).toMatch(/conflict|cannot be used/i);
});
it('dispatches dev ingest run through the low-level ingest Commander registration', async () => {
it('dispatches top-level ingest run through the low-level ingest Commander registration', async () => {
const io = makeIo();
const ingest = vi.fn(async () => 0);
await expect(
runKtxCli(
[
'dev',
'ingest',
'run',
'--connection-id',

13
packages/cli/src/dev.ts
View file

@ -1,11 +1,7 @@
import { resolve } from 'node:path';
import type { Command } from '@commander-js/extra-typings';
import { type CommandWithGlobalOptions, type KtxCliCommandContext, resolveCommandProjectDir } from './cli-program.js';
import { registerCompletionCommands } from './commands/completion-commands.js';
import { registerConnectionMappingCommands } from './commands/connection-commands.js';
import { registerIngestCommands } from './commands/ingest-commands.js';
import { registerRuntimeCommands } from './commands/runtime-commands.js';
import { registerScanCommands } from './commands/scan-commands.js';
import { profileMark } from './startup-profile.js';
profileMark('module:dev');
@ -13,7 +9,7 @@ profileMark('module:dev');
export function registerDevCommands(program: Command, context: KtxCliCommandContext): void {
const dev = program
.command('dev', { hidden: true })
.description('Low-level diagnostics, scans, adapter commands, and mapping tools')
.description('Low-level project initialization and runtime management')
.showHelpAfterError();
dev.hook('preAction', (_thisCommand, actionCommand) => {
@ -51,11 +47,4 @@ export function registerDevCommands(program: Command, context: KtxCliCommandCont
);
registerRuntimeCommands(dev, context);
registerScanCommands(dev, context);
registerIngestCommands(dev, context, {
runIngestWithProgress: async (ingestArgs, ingestIo, ingestDeps, defaultRunIngest) =>
await (ingestDeps.ingest ?? defaultRunIngest)(ingestArgs, ingestIo),
});
registerConnectionMappingCommands(dev, context);
registerCompletionCommands(dev, context, program);
}

38
packages/cli/src/example-smoke.test.ts
View file

@ -73,28 +73,23 @@ describe('standalone local warehouse example', () => {
const projectDir = await copyExampleProject(tempDir);
const sourceDir = join(projectDir, 'source');
const knowledgeList = await runBuiltCli(['agent', 'wiki', 'search', 'revenue', '--json', '--project-dir', projectDir]);
const knowledgeList = await runBuiltCli(['wiki', 'search', 'revenue', '--json', '--project-dir', projectDir]);
expect(knowledgeList).toMatchObject({ code: 0, stderr: '' });
expect(parseJsonOutput<{ results: Array<{ key: string; summary: string }> }>(knowledgeList.stdout).results).toContainEqual(
expect.objectContaining({ key: 'revenue', summary: 'Paid order value after refunds' }),
);
expect(
parseJsonOutput<{ data: { items: Array<{ key: string; summary: string }> } }>(knowledgeList.stdout).data.items,
).toContainEqual(expect.objectContaining({ key: 'revenue', summary: 'Paid order value after refunds' }));
const knowledgeRead = await runBuiltCli(['agent', 'wiki', 'read', 'revenue', '--json', '--project-dir', projectDir]);
expect(knowledgeRead).toMatchObject({ code: 0, stderr: '' });
expect(parseJsonOutput<{ content: string }>(knowledgeRead.stdout).content).toContain(
'Revenue is paid order amount after refund adjustments.',
);
const slList = await runBuiltCli(['agent', 'sl', 'list', '--json', '--project-dir', projectDir, '--connection-id', 'warehouse']);
const slList = await runBuiltCli(['sl', 'list', '--json', '--project-dir', projectDir, '--connection-id', 'warehouse']);
expect(slList).toMatchObject({ code: 0, stderr: '' });
expect(parseJsonOutput<{ sources: Array<{ connectionId: string; name: string; columnCount: number }> }>(slList.stdout).sources).toContainEqual(
expect.objectContaining({ connectionId: 'warehouse', name: 'orders', columnCount: 3 }),
);
expect(
parseJsonOutput<{ data: { items: Array<{ connectionId: string; name: string; columnCount: number }> } }>(
slList.stdout,
).data.items,
).toContainEqual(expect.objectContaining({ connectionId: 'warehouse', name: 'orders', columnCount: 3 }));
const slRead = await runBuiltCli([
'agent',
const slSearch = await runBuiltCli([
'sl',
'read',
'search',
'orders',
'--json',
'--connection-id',
@ -102,11 +97,12 @@ describe('standalone local warehouse example', () => {
'--project-dir',
projectDir,
]);
expect(slRead).toMatchObject({ code: 0, stderr: '' });
expect(parseJsonOutput<{ yaml: string }>(slRead.stdout).yaml).toContain('name: orders');
expect(slSearch).toMatchObject({ code: 0, stderr: '' });
expect(
parseJsonOutput<{ data: { items: Array<{ connectionId: string; name: string }> } }>(slSearch.stdout).data.items,
).toContainEqual(expect.objectContaining({ connectionId: 'warehouse', name: 'orders' }));
const ingest = await runBuiltCli([
'dev',
'ingest',
'run',
'--project-dir',
@ -120,7 +116,7 @@ describe('standalone local warehouse example', () => {
]);
expect(ingest).toMatchObject({ code: 1, stdout: '' });
expect(ingest.stderr).toContain(
'ktx dev ingest run requires llm.provider.backend: anthropic, vertex, or gateway, or an injected agentRunner',
'ktx ingest run requires llm.provider.backend: anthropic, vertex, or gateway, or an injected agentRunner',
);
}, 30_000);

1162
packages/cli/src/index.test.ts
View file

File diff suppressed because it is too large Load diff

11
packages/cli/src/index.ts
View file

@ -9,17 +9,6 @@ export {
type KtxCliIo,
type KtxCliPackageInfo,
} from './cli-runtime.js';
export { runKtxAgent, type KtxAgentArgs } from './agent.js';
export {
KTX_AGENT_MAX_ROWS_CAP,
createKtxAgentRuntime,
parseAgentMaxRows,
readAgentJsonFile,
writeAgentJson,
writeAgentJsonError,
type KtxAgentRuntime,
type KtxAgentRuntimeDeps,
} from './agent-runtime.js';
export { runKtxSetup, type KtxSetupArgs, type KtxSetupStatus } from './setup.js';
export type {
KtxSetupDatabaseDriver,

86
packages/cli/src/ingest-query-executor.test.ts Normal file
View file

@ -0,0 +1,86 @@
import type { KtxLocalProject } from '@ktx/context/project';
import { createKtxConnectorCapabilities, type KtxScanConnector } from '@ktx/context/scan';
import { describe, expect, it, vi } from 'vitest';
import { createKtxCliIngestQueryExecutor } from './ingest-query-executor.js';
function project(): KtxLocalProject {
return {
projectDir: '/tmp/ktx-query-project',
config: {
project: 'warehouse',
connections: {
warehouse: { driver: 'postgres', url: 'postgresql://readonly@example.test/db' },
},
},
} as unknown as KtxLocalProject;
}
function connector(overrides: Partial<KtxScanConnector> = {}): KtxScanConnector {
return {
id: 'warehouse',
driver: 'postgres',
capabilities: createKtxConnectorCapabilities({ readOnlySql: true }),
async introspect() {
throw new Error('introspect is not used by this test');
},
executeReadOnly: vi.fn(async () => ({
headers: ['answer'],
rows: [[1]],
totalRows: 1,
rowCount: 1,
})),
cleanup: vi.fn(async () => {}),
...overrides,
};
}
describe('createKtxCliIngestQueryExecutor', () => {
it('executes read-only SQL through the scan connector and cleans it up', async () => {
const scanConnector = connector();
const createConnector = vi.fn(async () => scanConnector);
const executor = createKtxCliIngestQueryExecutor(project(), { createConnector });
await expect(
executor.execute({
connectionId: 'warehouse',
connection: { driver: 'postgres', url: 'postgresql://readonly@example.test/db' },
projectDir: '/tmp/ktx-query-project',
sql: 'select 1',
maxRows: 5,
}),
).resolves.toMatchObject({
headers: ['answer'],
rows: [[1]],
totalRows: 1,
command: 'SELECT',
rowCount: 1,
});
expect(createConnector).toHaveBeenCalledWith(project(), 'warehouse');
expect(scanConnector.executeReadOnly).toHaveBeenCalledWith(
{ connectionId: 'warehouse', sql: 'select 1', maxRows: 5 },
{ runId: 'ingest-sql-execution' },
);
expect(scanConnector.cleanup).toHaveBeenCalledTimes(1);
});
it('rejects connectors without read-only SQL support', async () => {
const scanConnector = connector({
capabilities: createKtxConnectorCapabilities({ readOnlySql: false }),
executeReadOnly: undefined,
});
const executor = createKtxCliIngestQueryExecutor(project(), {
createConnector: vi.fn(async () => scanConnector),
});
await expect(
executor.execute({
connectionId: 'warehouse',
connection: { driver: 'postgres' },
projectDir: '/tmp/ktx-query-project',
sql: 'select 1',
}),
).rejects.toThrow('Connection "warehouse" driver "postgres" does not support read-only SQL execution.');
expect(scanConnector.cleanup).toHaveBeenCalledTimes(1);
});
});

49
packages/cli/src/ingest-query-executor.ts Normal file
View file

@ -0,0 +1,49 @@
import type { KtxSqlQueryExecutionInput, KtxSqlQueryExecutorPort } from '@ktx/context/connections';
import type { KtxLocalProject } from '@ktx/context/project';
import type { KtxScanConnector, KtxScanContext } from '@ktx/context/scan';
import { createKtxCliScanConnector } from './local-scan-connectors.js';
type CreateConnector = typeof createKtxCliScanConnector;
export interface KtxCliIngestQueryExecutorDeps {
createConnector?: CreateConnector;
}
async function cleanupConnector(connector: KtxScanConnector | null): Promise<void> {
await connector?.cleanup?.();
}
export function createKtxCliIngestQueryExecutor(
project: KtxLocalProject,
deps: KtxCliIngestQueryExecutorDeps = {},
): KtxSqlQueryExecutorPort {
const createConnector = deps.createConnector ?? createKtxCliScanConnector;
return {
async execute(input: KtxSqlQueryExecutionInput) {
let connector: KtxScanConnector | null = null;
try {
connector = await createConnector(project, input.connectionId);
if (!connector.capabilities.readOnlySql || !connector.executeReadOnly) {
throw new Error(
`Connection "${input.connectionId}" driver "${connector.driver}" does not support read-only SQL execution.`,
);
}
const ctx: KtxScanContext = { runId: 'ingest-sql-execution' };
const result = await connector.executeReadOnly(
{ connectionId: input.connectionId, sql: input.sql, maxRows: input.maxRows },
ctx,
);
return {
headers: result.headers,
rows: result.rows,
totalRows: result.totalRows,
command: 'SELECT',
rowCount: result.rowCount,
};
} finally {
await cleanupConnector(connector);
}
},
};
}

80
packages/cli/src/ingest.test-utils.ts
View file

@ -1,23 +1,20 @@
import { EventEmitter } from 'node:events';
import { access, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { mkdir, writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import { AgentRunnerService, type RunLoopParams } from '@ktx/context/agent';
import {
LocalLookerRuntimeStore,
LocalMetabaseSourceStateReader,
KtxYamlMetabaseSourceStateReader,
LocalMetabaseDiscoveryCache,
MetabaseSourceAdapter,
getLocalIngestStatus,
type ChunkResult,
type FetchContext,
type IngestReportSnapshot,
type LocalIngestResult,
type LocalMetabaseFanoutProgress,
type LookerMappingClient,
type LookerRuntimeClient,
type LookerTableIdentifierParser,
type MemoryFlowEventSink,
type MemoryFlowReplayInput,
type MetabaseCard,
type MetabaseCardSummary,
type MetabaseClientFactory,
@ -28,7 +25,7 @@ import {
} from '@ktx/context/ingest';
import { ktxLocalStateDbPath, loadKtxProject } from '@ktx/context/project';
import { expect, vi } from 'vitest';
import { type KtxIngestArgs, runKtxIngest } from './ingest.js';
import { runKtxIngest } from './ingest.js';
export function makeIo(
options: {
@ -162,7 +159,7 @@ export function bundleReportSnapshot(): IngestReportSnapshot {
rawFiles: ['cards/1.json', 'cards/2.json'],
status: 'success',
actions: [
{ target: 'wiki', type: 'created', key: 'knowledge/global/revenue.md', detail: 'Revenue overview' },
{ target: 'wiki', type: 'created', key: 'wiki/global/revenue.md', detail: 'Revenue overview' },
{ target: 'sl', type: 'updated', key: 'warehouse.orders', detail: 'Added order amount measure' },
],
touchedSlSources: [{ connectionId: 'warehouse', sourceName: 'warehouse.orders' }],
@ -181,7 +178,7 @@ export function bundleReportSnapshot(): IngestReportSnapshot {
{
rawPath: 'cards/1.json',
artifactKind: 'wiki',
artifactKey: 'knowledge/global/revenue.md',
artifactKey: 'wiki/global/revenue.md',
actionType: 'wiki_written',
},
{
@ -197,7 +194,7 @@ export function bundleReportSnapshot(): IngestReportSnapshot {
path: 'tool-transcripts/cards.jsonl',
toolCallCount: 4,
errorCount: 0,
toolNames: ['ingest_triage', 'knowledge_capture', 'sl_capture'],
toolNames: ['ingest_triage', 'wiki_capture', 'sl_capture'],
},
],
},
@ -265,6 +262,18 @@ export class CliLookerSlWritingAgentRunner extends AgentRunnerService {
params.telemetryTags?.operationName === 'ingest-bundle-wu' &&
params.telemetryTags?.unitKey === 'looker-explore-ecommerce-orders'
) {
const ledger = params.toolSet.record_verification_ledger;
if (!ledger?.execute) {
throw new Error('record_verification_ledger tool was not available to the Looker WorkUnit');
}
await ledger.execute(
{
summary: 'Test fixture verified Looker explore target identifiers before writing SL.',
verifiedIdentifiers: ['prod-warehouse', 'public.orders'],
unverifiedIdentifiers: [],
},
{ toolCallId: 'cli-looker-verification-ledger', messages: [] },
);
const slWrite = params.toolSet.sl_write_source;
if (!slWrite?.execute) {
throw new Error('sl_write_source tool was not available to the Looker WorkUnit');
@ -367,7 +376,7 @@ const SYNC_MODE_METABASE_CARDS: MetabaseCard[] = [
collection_id: 12,
archived: false,
result_metadata: [],
dataset_query: { type: 'native', database: 1, native: { query: 'select 101 as id' } },
dataset_query: { type: 'native', database: 1, stages: [{ 'lib/type': 'mbql.stage/native', native: 'select 101 as id' }] },
parameters: [],
dashboard_count: 0,
},
@ -381,7 +390,7 @@ const SYNC_MODE_METABASE_CARDS: MetabaseCard[] = [
collection_id: 12,
archived: false,
result_metadata: [],
dataset_query: { type: 'native', database: 1, native: { query: 'select 102 as id' } },
dataset_query: { type: 'native', database: 1, stages: [{ 'lib/type': 'mbql.stage/native', native: 'select 102 as id' }] },
parameters: [],
dashboard_count: 0,
},
@ -395,7 +404,7 @@ const SYNC_MODE_METABASE_CARDS: MetabaseCard[] = [
collection_id: 13,
archived: false,
result_metadata: [],
dataset_query: { type: 'native', database: 1, native: { query: 'select 103 as id' } },
dataset_query: { type: 'native', database: 1, stages: [{ 'lib/type': 'mbql.stage/native', native: 'select 103 as id' }] },
parameters: [],
dashboard_count: 0,
},
@ -445,11 +454,11 @@ function createSyncModeMetabaseClient(): MetabaseRuntimeClient {
},
getAllCards: async () => SYNC_MODE_METABASE_CARDS.map(metabaseCardSummary),
convertMbqlToNative: async () => ({ query: 'select 1' }),
getNativeSql: (card) => card.dataset_query?.native?.query ?? null,
getNativeSql: (card) => card.dataset_query?.stages?.[0]?.native ?? null,
getTemplateTags: () => ({}),
getCardSql: async (card) => card.dataset_query?.native?.query ?? null,
getCardSql: async (card) => card.dataset_query?.stages?.[0]?.native ?? null,
getResolvedSql: async (card) => ({
resolvedSql: card.dataset_query?.native?.query ?? `select ${card.id} as id`,
resolvedSql: card.dataset_query?.stages?.[0]?.native ?? `select ${card.id} as id`,
templateTags: [],
resolutionStatus: 'resolved',
}),
@ -485,6 +494,23 @@ export async function runPublicMetabaseSyncModeCase(tempDir: string, input: Sync
' driver: metabase',
' api_url: https://metabase.example.test',
' api_key: literal-test-key',
' mappings:',
' databaseMappings:',
' "1": warehouse_a',
' syncEnabled:',
' "1": true',
` syncMode: ${input.syncMode}`,
' selections:',
` collections: [${input.selections
.filter((selection) => selection.selectionType === 'collection')
.map((selection) => selection.metabaseObjectId)
.join(', ')}]`,
` items: [${input.selections
.filter((selection) => selection.selectionType === 'item')
.map((selection) => selection.metabaseObjectId)
.join(', ')}]`,
' defaultTagNames:',
' - sync-mode-smoke',
' warehouse_a:',
' driver: postgres',
' url: postgresql://readonly@db.example.test/warehouse_a',
@ -499,29 +525,15 @@ export async function runPublicMetabaseSyncModeCase(tempDir: string, input: Sync
);
const project = await loadKtxProject({ projectDir });
const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(project) });
await store.replaceSourceState({
const discoveryCache = new LocalMetabaseDiscoveryCache({ dbPath: ktxLocalStateDbPath(project) });
await discoveryCache.refreshDiscoveredDatabases({
connectionId: 'prod-metabase',
syncMode: input.syncMode,
defaultTagNames: ['sync-mode-smoke'],
selections: input.selections,
mappings: [
{
metabaseDatabaseId: 1,
metabaseDatabaseName: 'Warehouse A',
metabaseEngine: 'postgres',
metabaseHost: 'db.example.test',
metabaseDbName: 'warehouse_a',
targetConnectionId: 'warehouse_a',
syncEnabled: true,
source: 'refresh',
},
],
discovered: [{ id: 1, name: 'Warehouse A', engine: 'postgres', host: 'db.example.test', dbName: 'warehouse_a' }],
});
const adapter = new MetabaseSourceAdapter({
clientFactory: new StaticMetabaseClientFactory(createSyncModeMetabaseClient()),
sourceStateReader: store,
sourceStateReader: new KtxYamlMetabaseSourceStateReader(project, { discoveryCache }),
});
const jobId = `metabase-sync-mode-${input.name}-child`;
const io = makeIo();

235
packages/cli/src/ingest.test.ts
View file

@ -3,11 +3,9 @@ import { tmpdir } from 'node:os';
import { join } from 'node:path';
import {
LocalLookerRuntimeStore,
LocalMetabaseSourceStateReader,
getLocalIngestStatus,
LocalMetabaseDiscoveryCache,
type LocalIngestResult,
type LocalMetabaseFanoutProgress,
type MemoryFlowReplayInput,
type RunLocalIngestOptions,
type SourceAdapter,
} from '@ktx/context/ingest';
@ -20,7 +18,6 @@ import {
CliMetabaseAgentRunner,
CliMetabaseSourceAdapter,
completedLocalBundleRun,
emitLiveLocalMemoryFlow,
failedLocalBundleRun,
localFakeBundleReport,
makeCliLookerParser,
@ -28,7 +25,6 @@ import {
makeIo,
persistLocalBundleReport,
runPublicMetabaseSyncModeCase,
writeBundleReportFile,
writeMetabaseConfig,
writeWarehouseConfig,
} from './ingest.test-utils.js';
@ -107,7 +103,89 @@ describe('runKtxIngest', () => {
expect(statusIo.stderr()).toBe('');
});
it('prints provider setup guidance when a skip-llm setup project runs dev ingest', async () => {
it('emits structured progress for non-TTY local ingest runs', async () => {
const projectDir = join(tempDir, 'project');
await writeWarehouseConfig(projectDir);
const progressEvents: Array<{ percent: number; message: string; transient?: boolean }> = [];
const runLocal = vi.fn(async (input: RunLocalIngestOptions): Promise<LocalIngestResult> => {
input.memoryFlow?.emit({ type: 'source_acquired', adapter: 'fake', trigger: 'manual_resync', fileCount: 2 });
input.memoryFlow?.emit({ type: 'chunks_planned', chunkCount: 2, workUnitCount: 2, evictionCount: 0 });
input.memoryFlow?.emit({ type: 'work_unit_started', unitKey: 'orders', skills: [], stepBudget: 4 });
input.memoryFlow?.emit({ type: 'work_unit_step', unitKey: 'orders', stepIndex: 2, stepBudget: 4 });
return completedLocalBundleRun(input, 'cli-local-progress-1');
});
const io = makeIo();
await expect(
runKtxIngest(
{
command: 'run',
projectDir,
connectionId: 'warehouse',
adapter: 'fake',
outputMode: 'plain',
},
io.io,
{
runLocalIngest: runLocal,
jobIdFactory: () => 'cli-local-progress-1',
progress: (event) => progressEvents.push(event),
},
),
).resolves.toBe(0);
expect(progressEvents).toEqual(
expect.arrayContaining([
{ percent: 5, message: 'Fetching source files for warehouse/fake' },
{ percent: 15, message: 'Fetched 2 source files from fake' },
{ percent: 45, message: 'Planned 2 work units' },
expect.objectContaining({
message: 'Processing work units: 0/2 complete, 1 active; latest orders step 2/4',
transient: true,
}),
]),
);
expect(io.stderr()).not.toContain('[15%] Fetched 2 source files from fake');
});
it('describes zero-work-unit ingest progress as finalizing instead of appearing half-planned', async () => {
const projectDir = join(tempDir, 'project');
await writeWarehouseConfig(projectDir);
const progressEvents: Array<{ percent: number; message: string; transient?: boolean }> = [];
const runLocal = vi.fn(async (input: RunLocalIngestOptions): Promise<LocalIngestResult> => {
input.memoryFlow?.emit({ type: 'source_acquired', adapter: 'fake', trigger: 'manual_resync', fileCount: 2 });
input.memoryFlow?.emit({ type: 'chunks_planned', chunkCount: 0, workUnitCount: 0, evictionCount: 0 });
return completedLocalBundleRun(input, 'cli-local-zero-progress-1');
});
const io = makeIo();
await expect(
runKtxIngest(
{
command: 'run',
projectDir,
connectionId: 'warehouse',
adapter: 'fake',
outputMode: 'plain',
},
io.io,
{
runLocalIngest: runLocal,
jobIdFactory: () => 'cli-local-zero-progress-1',
progress: (event) => progressEvents.push(event),
},
),
).resolves.toBe(0);
expect(progressEvents).toEqual(
expect.arrayContaining([
{ percent: 80, message: 'No work units to process; finalizing ingest' },
]),
);
expect(progressEvents).not.toContainEqual({ percent: 45, message: 'Planned 0 work units' });
});
it('prints provider setup guidance when a skip-llm setup project runs ingest', async () => {
const projectDir = join(tempDir, 'project');
const setupIo = makeIo();
await expect(
@ -168,7 +246,7 @@ describe('runKtxIngest', () => {
expect(runIo.stdout()).toBe('');
expect(runIo.stderr()).toContain(
'ktx dev ingest run requires llm.provider.backend: anthropic, vertex, or gateway, or an injected agentRunner.',
'ktx ingest run requires llm.provider.backend: anthropic, vertex, or gateway, or an injected agentRunner.',
);
expect(runIo.stderr()).toContain(
`ktx setup --project-dir ${projectDir} --anthropic-api-key-env ANTHROPIC_API_KEY --anthropic-model claude-sonnet-4-6 --no-input`,
@ -425,6 +503,65 @@ describe('runKtxIngest', () => {
expect(io.stdout()).not.toContain('status=running job=metabase-child-1');
});
it('emits structured progress for Metabase fan-out without writing progress to JSON output', async () => {
const projectDir = join(tempDir, 'project');
await writeMetabaseConfig(projectDir);
const io = makeIo();
const progressEvents: Array<{ percent: number; message: string }> = [];
await expect(
runKtxIngest(
{
command: 'run',
projectDir,
connectionId: 'prod-metabase',
adapter: 'metabase',
outputMode: 'json',
},
io.io,
{
progress: (event) => progressEvents.push(event),
runLocalMetabaseIngest: async (input) => {
input.progress?.onMetabaseFanoutPlanned?.({
metabaseConnectionId: 'prod-metabase',
children: [{ metabaseDatabaseId: 1, targetConnectionId: 'warehouse_a' }],
});
input.progress?.onMetabaseChildStarted?.({
metabaseConnectionId: 'prod-metabase',
metabaseDatabaseId: 1,
targetConnectionId: 'warehouse_a',
jobId: 'metabase-child-1',
});
input.progress?.onMetabaseChildCompleted?.({
metabaseConnectionId: 'prod-metabase',
metabaseDatabaseId: 1,
targetConnectionId: 'warehouse_a',
jobId: 'metabase-child-1',
status: 'done',
});
return {
metabaseConnectionId: 'prod-metabase',
status: 'all_succeeded',
totals: { workUnits: 0, failedWorkUnits: 0 },
children: [],
};
},
},
),
).resolves.toBe(0);
expect(progressEvents).toEqual(
expect.arrayContaining([
{ percent: 5, message: 'Checking Metabase mappings for prod-metabase' },
{ percent: 10, message: 'Metabase prod-metabase: 1 mapped database' },
{ percent: 25, message: 'Metabase database 1 -> warehouse_a running' },
{ percent: 90, message: 'Metabase database 1 -> warehouse_a done' },
]),
);
expect(io.stdout()).toContain('"status": "all_succeeded"');
expect(io.stderr()).not.toContain('Metabase ingest: prod-metabase');
});
it('runs Metabase scheduled ingest through the public CLI command path with real fan-out', async () => {
const projectDir = join(tempDir, 'metabase-cli-project');
await writeWarehouseConfig(projectDir);
@ -437,6 +574,16 @@ describe('runKtxIngest', () => {
' driver: metabase',
' api_url: https://metabase.example.test',
' api_key: literal-test-key',
' mappings:',
' databaseMappings:',
' "1": warehouse_a',
' "2": warehouse_b',
' syncEnabled:',
' "1": true',
' "2": true',
' syncMode: ALL',
' defaultTagNames:',
' - ktx',
' warehouse_a:',
' driver: postgres',
' url: postgresql://readonly@db.example.test/warehouse_a',
@ -453,33 +600,12 @@ describe('runKtxIngest', () => {
'utf-8',
);
const project = await loadKtxProject({ projectDir });
const store = new LocalMetabaseSourceStateReader({ dbPath: ktxLocalStateDbPath(project) });
await store.replaceSourceState({
const discoveryCache = new LocalMetabaseDiscoveryCache({ dbPath: ktxLocalStateDbPath(project) });
await discoveryCache.refreshDiscoveredDatabases({
connectionId: 'prod-metabase',
syncMode: 'ALL',
defaultTagNames: ['ktx'],
selections: [],
mappings: [
{
metabaseDatabaseId: 1,
metabaseDatabaseName: 'Warehouse A',
metabaseEngine: 'postgres',
metabaseHost: 'db.example.test',
metabaseDbName: 'warehouse_a',
targetConnectionId: 'warehouse_a',
syncEnabled: true,
source: 'refresh',
},
{
metabaseDatabaseId: 2,
metabaseDatabaseName: 'Warehouse B',
metabaseEngine: 'postgres',
metabaseHost: 'db.example.test',
metabaseDbName: 'warehouse_b',
targetConnectionId: 'warehouse_b',
syncEnabled: true,
source: 'refresh',
},
discovered: [
{ id: 1, name: 'Warehouse A', engine: 'postgres', host: 'db.example.test', dbName: 'warehouse_a' },
{ id: 2, name: 'Warehouse B', engine: 'postgres', host: 'db.example.test', dbName: 'warehouse_b' },
],
});
const adapter = new CliMetabaseSourceAdapter();
@ -663,7 +789,7 @@ describe('runKtxIngest', () => {
).resolves.toBe(1);
expect(io.stderr()).toContain('source-dir uploads are not supported for the Metabase fan-out adapter');
expect(io.stderr()).not.toContain('ktx dev ingest run requires llm.provider.backend');
expect(io.stderr()).not.toContain('ktx ingest run requires llm.provider.backend');
expect(io.stdout()).toBe('');
});
@ -720,7 +846,6 @@ describe('runKtxIngest', () => {
patternPagesWritten: 30,
stalePatternPagesMarked: 2,
archivedPatternPages: 3,
legacyPagesDeleted: 4,
},
errors: [],
warnings: [],
@ -754,7 +879,7 @@ describe('runKtxIngest', () => {
expect(io.stderr()).toBe('');
expect(io.stdout()).toContain('Adapter: historic-sql\n');
expect(io.stdout()).toContain('Saved memory: 39 wiki, 57 SL\n');
expect(io.stdout()).toContain('Saved memory: 35 wiki, 57 SL\n');
});
it('returns a non-zero code when local ingest reports failed work units', async () => {
@ -814,6 +939,44 @@ describe('runKtxIngest', () => {
expect(runLocalIngest).toHaveBeenCalledWith(expect.objectContaining({ llmDebugRequestFile: debugFile }));
});
it('supplies a scan-connector query executor to local ingest runs', async () => {
const io = makeIo();
const projectDir = join(tempDir, 'query-executor-project');
await writeWarehouseConfig(projectDir);
const queryExecutor = {
execute: vi.fn(async () => ({
headers: [],
rows: [],
totalRows: 0,
command: 'SELECT',
rowCount: 0,
})),
};
const runLocalIngest = vi.fn(async (input: RunLocalIngestOptions): Promise<LocalIngestResult> =>
completedLocalBundleRun(input, 'query-executor-run'),
);
await expect(
runKtxIngest(
{
command: 'run',
projectDir,
connectionId: 'warehouse',
adapter: 'fake',
outputMode: 'json',
},
io.io,
{
runLocalIngest,
createAdapters: () => [],
createQueryExecutor: () => queryExecutor,
},
),
).resolves.toBe(0);
expect(runLocalIngest).toHaveBeenCalledWith(expect.objectContaining({ queryExecutor }));
});
it('passes daemon database introspection URL to default local ingest adapters', async () => {
const projectDir = join(tempDir, 'project');
await writeWarehouseConfig(projectDir);

127
packages/cli/src/ingest.ts
View file

@ -16,7 +16,9 @@ import {
runLocalMetabaseIngest,
savedMemoryCountsForReport,
} from '@ktx/context/ingest';
import { loadKtxProject } from '@ktx/context/project';
import type { KtxSqlQueryExecutorPort } from '@ktx/context/connections';
import { loadKtxProject, type KtxLocalProject } from '@ktx/context/project';
import { createKtxCliIngestQueryExecutor } from './ingest-query-executor.js';
import { readIngestReportSnapshotFile } from './ingest-report-file.js';
import { createCliOperationalLogger } from './io/logger.js';
import { createKtxCliLocalIngestAdapters } from './local-adapters.js';
@ -65,10 +67,17 @@ interface KtxIngestIo {
stderr: { write(chunk: string): void };
}
interface KtxIngestDeps {
export interface KtxIngestProgressUpdate {
percent: number;
message: string;
transient?: boolean;
}
export interface KtxIngestDeps {
jobIdFactory?: () => string;
now?: () => Date;
createAdapters?: typeof createKtxCliLocalIngestAdapters;
createQueryExecutor?: (project: KtxLocalProject) => KtxSqlQueryExecutorPort;
runLocalIngest?: typeof runLocalIngest;
runLocalMetabaseIngest?: typeof runLocalMetabaseIngest;
readReportFile?: typeof readIngestReportSnapshotFile;
@ -85,6 +94,7 @@ interface KtxIngestDeps {
| 'logger'
| 'pullConfigOptions'
>;
progress?: (update: KtxIngestProgressUpdate) => void;
}
function reportStatus(report: IngestReportSnapshot): 'done' | 'error' {
@ -142,12 +152,18 @@ function pluralize(count: number, singular: string, plural = `${singular}s`): st
function createMetabaseFanoutProgress(
connectionId: string,
io: KtxIngestIo,
onProgress?: (update: KtxIngestProgressUpdate) => void,
): LocalMetabaseFanoutProgress {
io.stderr.write(`Metabase ingest: ${connectionId}\n`);
io.stderr.write('Checking mappings and scheduled-pull targets...\n');
onProgress?.({ percent: 5, message: `Checking Metabase mappings for ${connectionId}` });
return {
onMetabaseFanoutPlanned(event) {
io.stderr.write(`Targets: ${pluralize(event.children.length, 'mapped database')}\n`);
onProgress?.({
percent: 10,
message: `Metabase ${event.metabaseConnectionId}: ${pluralize(event.children.length, 'mapped database')}`,
});
for (const child of event.children) {
io.stderr.write(`- database=${child.metabaseDatabaseId} target=${child.targetConnectionId} status=queued\n`);
}
@ -156,11 +172,19 @@ function createMetabaseFanoutProgress(
io.stderr.write(
`- database=${event.metabaseDatabaseId} target=${event.targetConnectionId} status=running job=${event.jobId}\n`,
);
onProgress?.({
percent: 25,
message: `Metabase database ${event.metabaseDatabaseId} -> ${event.targetConnectionId} running`,
});
},
onMetabaseChildCompleted(event) {
io.stderr.write(
`- database=${event.metabaseDatabaseId} target=${event.targetConnectionId} status=${event.status} job=${event.jobId}\n`,
);
onProgress?.({
percent: 90,
message: `Metabase database ${event.metabaseDatabaseId} -> ${event.targetConnectionId} ${event.status}`,
});
},
};
}
@ -228,6 +252,12 @@ function plainIngestEventProgress(
case 'diff_computed':
return { percent: 35, message: `Computed source diff ${formatDiffProgress(event)}` };
case 'chunks_planned':
if (event.workUnitCount === 0) {
return {
percent: 80,
message: 'No work units to process; finalizing ingest',
};
}
return {
percent: 45,
message: `Planned ${pluralize(event.workUnitCount, 'work unit')}`,
@ -293,34 +323,22 @@ function shouldWritePlainIngestProgress(
return outputMode === 'plain' && io.stdout.isTTY === true && env.CI !== 'true';
}
function createPlainIngestProgressRenderer(
function createPlainIngestProgressObserver(
args: Extract<KtxIngestArgs, { command: 'run' }>,
io: KtxIngestIo,
): { start(): void; update(snapshot: MemoryFlowReplayInput): void; flush(): void } {
onProgress: (update: KtxIngestProgressUpdate) => void,
): { start(): void; update(snapshot: MemoryFlowReplayInput): void } {
let printedEvents = 0;
let lastPercent = 0;
let printedCompletion = false;
let hasPendingTransient = false;
const flush = () => {
if (!hasPendingTransient) {
return;
}
io.stderr.write('\n');
hasPendingTransient = false;
};
const write = (percent: number, message: string, options?: { transient?: boolean }) => {
const nextPercent = Math.max(lastPercent, Math.max(0, Math.min(100, percent)));
lastPercent = nextPercent;
const line = `[${nextPercent}%] ${message}`;
if (options?.transient === true) {
io.stderr.write(`\r${line}\u001b[K`);
hasPendingTransient = true;
return;
}
flush();
io.stderr.write(`${line}\n`);
onProgress({
percent: nextPercent,
message,
...(options?.transient !== undefined ? { transient: options.transient } : {}),
});
};
return {
@ -344,6 +362,41 @@ function createPlainIngestProgressRenderer(
write(100, snapshot.status === 'done' ? 'Ingest completed' : 'Ingest failed');
}
},
};
}
function createPlainIngestProgressRenderer(
args: Extract<KtxIngestArgs, { command: 'run' }>,
io: KtxIngestIo,
): { start(): void; update(snapshot: MemoryFlowReplayInput): void; flush(): void } {
let hasPendingTransient = false;
const flush = () => {
if (!hasPendingTransient) {
return;
}
io.stderr.write('\n');
hasPendingTransient = false;
};
const observer = createPlainIngestProgressObserver(args, (update) => {
const line = `[${update.percent}%] ${update.message}`;
if (update.transient === true) {
io.stderr.write(`\r${line}\u001b[K`);
hasPendingTransient = true;
return;
}
flush();
io.stderr.write(`${line}\n`);
});
return {
start() {
observer.start();
},
update(snapshot) {
observer.update(snapshot);
},
flush,
};
}
@ -518,7 +571,9 @@ export async function runKtxIngest(
const project = await loadKtxProject({ projectDir: args.projectDir });
const env = deps.env ?? process.env;
if (args.command === 'run') {
const createAdapters = deps.createAdapters ?? createKtxCliLocalIngestAdapters;
const createAdapters =
deps.createAdapters ??
(deps.runLocalIngest || deps.runLocalMetabaseIngest ? () => [] : createKtxCliLocalIngestAdapters);
const executeLocalIngest = deps.runLocalIngest ?? runLocalIngest;
const localIngestOptions = deps.localIngestOptions ?? {};
const managedDaemon = managedDaemonOptionsForIngestRun(args, io);
@ -530,18 +585,30 @@ export async function runKtxIngest(
...(args.adapter === 'historic-sql' ? { historicSqlConnectionId: args.connectionId } : {}),
logger: operationalLogger,
};
const queryExecutor =
localIngestOptions.queryExecutor ??
(deps.createQueryExecutor ?? createKtxCliIngestQueryExecutor)(project);
if (args.adapter === 'metabase' && args.sourceDir) {
throw new Error('source-dir uploads are not supported for the Metabase fan-out adapter');
}
if (args.adapter === 'metabase') {
const executeMetabaseFanout = deps.runLocalMetabaseIngest ?? runLocalMetabaseIngest;
const progress =
args.outputMode === 'json' ? undefined : createMetabaseFanoutProgress(args.connectionId, io);
args.outputMode === 'json' && !deps.progress
? undefined
: createMetabaseFanoutProgress(
args.connectionId,
args.outputMode === 'json'
? { ...io, stderr: { write: () => undefined } }
: io,
deps.progress,
);
const result = await executeMetabaseFanout({
project,
adapters: createAdapters(project, adapterOptions),
metabaseConnectionId: args.connectionId,
...localIngestOptions,
queryExecutor,
trigger: 'manual_resync',
jobIdFactory: deps.jobIdFactory,
...(progress ? { progress } : {}),
@ -564,8 +631,13 @@ export async function runKtxIngest(
const plainProgress = shouldWritePlainIngestProgress(runOutputMode, io, env)
? createPlainIngestProgressRenderer(args, io)
: null;
const structuredProgress = deps.progress
? createPlainIngestProgressObserver(args, deps.progress)
: null;
const initialMemoryFlow =
shouldUseLiveViz || plainProgress ? initialRunMemoryFlowInput(args, jobId ?? 'pending') : undefined;
shouldUseLiveViz || plainProgress || structuredProgress
? initialRunMemoryFlowInput(args, jobId ?? 'pending')
: undefined;
let latestMemoryFlowSnapshot: MemoryFlowReplayInput | null = initialMemoryFlow ?? null;
if (shouldUseLiveViz && initialMemoryFlow && isTuiCapableIo(io)) {
@ -586,11 +658,13 @@ export async function runKtxIngest(
return;
}
plainProgress?.update(snapshot);
structuredProgress?.update(snapshot);
},
})
: undefined;
plainProgress?.start();
structuredProgress?.start();
try {
const result = await executeLocalIngest({
@ -602,6 +676,7 @@ export async function runKtxIngest(
trigger: 'manual_resync',
jobId,
...localIngestOptions,
queryExecutor,
pullConfigOptions: adapterOptions,
...(args.debugLlmRequestFile ? { llmDebugRequestFile: args.debugLlmRequestFile } : {}),
...(memoryFlow ? { memoryFlow } : {}),
@ -645,7 +720,7 @@ export async function runKtxIngest(
throw new Error(
args.runId
? `Local ingest run or report "${args.runId}" was not found`
: 'No local ingest reports were found. Run `ktx ingest --all` first.',
: 'No local ingest reports were found. Run `ktx ingest run --connection-id <id> --adapter <adapter>` first.',
);
}
await writeReportRecord(report, args.outputMode, io, {

63
packages/cli/src/knowledge.test.ts
View file

@ -51,7 +51,7 @@ describe('runKtxKnowledge', () => {
await rm(tempDir, { recursive: true, force: true });
});
it('writes, reads, lists, and searches knowledge pages', async () => {
it('writes, reads, lists, and searches wiki pages', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
@ -73,7 +73,7 @@ describe('runKtxKnowledge', () => {
writeIo.io,
),
).resolves.toBe(0);
expect(writeIo.stdout()).toContain('Wrote knowledge/global/metrics-revenue.md');
expect(writeIo.stdout()).toContain('Wrote wiki/global/metrics-revenue.md');
const readIo = makeIo();
await expect(
@ -93,6 +93,65 @@ describe('runKtxKnowledge', () => {
expect(searchIo.stdout()).toContain('metrics-revenue');
});
it('prints wiki list, search, and read as public JSON envelopes', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
await expect(
runKtxKnowledge(
{
command: 'write',
projectDir,
key: 'metrics-revenue',
scope: 'GLOBAL',
userId: 'local',
summary: 'Revenue',
content: 'Revenue is paid order value.',
tags: ['finance'],
refs: [],
slRefs: ['orders'],
},
makeIo().io,
),
).resolves.toBe(0);
const listIo = makeIo();
await expect(runKtxKnowledge({ command: 'list', projectDir, userId: 'local', json: true }, listIo.io)).resolves.toBe(
0,
);
expect(JSON.parse(listIo.stdout())).toMatchObject({
kind: 'list',
data: { items: [expect.objectContaining({ key: 'metrics-revenue', summary: 'Revenue' })] },
meta: { command: 'wiki list' },
});
const searchIo = makeIo();
await expect(
runKtxKnowledge(
{ command: 'search', projectDir, query: 'paid order', userId: 'local', json: true, limit: 5 },
searchIo.io,
),
).resolves.toBe(0);
expect(JSON.parse(searchIo.stdout())).toMatchObject({
kind: 'list',
data: { items: [expect.objectContaining({ key: 'metrics-revenue', summary: 'Revenue' })] },
meta: { command: 'wiki search' },
});
const readIo = makeIo();
await expect(
runKtxKnowledge({ command: 'read', projectDir, key: 'metrics-revenue', userId: 'local', json: true }, readIo.io),
).resolves.toBe(0);
expect(JSON.parse(readIo.stdout())).toMatchObject({
kind: 'wiki.page',
data: {
key: 'metrics-revenue',
summary: 'Revenue',
content: 'Revenue is paid order value.',
},
});
});
it('rejects slash-delimited write keys with a flat-key suggestion', async () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });

34
packages/cli/src/knowledge.ts
View file

@ -11,11 +11,12 @@ import {
searchLocalKnowledgePages,
writeLocalKnowledgePage,
} from '@ktx/context/wiki';
import { writeJsonResult } from './io/print-list.js';
export type KtxKnowledgeArgs =
| { command: 'list'; projectDir: string; userId: string }
| { command: 'read'; projectDir: string; key: string; userId: string }
| { command: 'search'; projectDir: string; query: string; userId: string }
| { command: 'list'; projectDir: string; userId: string; json?: boolean }
| { command: 'read'; projectDir: string; key: string; userId: string; json?: boolean }
| { command: 'search'; projectDir: string; query: string; userId: string; json?: boolean; limit?: number }
| {
command: 'write';
projectDir: string;
@ -61,6 +62,14 @@ export async function runKtxKnowledge(
const project = await loadKtxProject({ projectDir: args.projectDir });
if (args.command === 'list') {
const pages = await listLocalKnowledgePages(project, { userId: args.userId });
if (args.json) {
writeJsonResult(io, {
kind: 'list',
data: { items: pages },
meta: { command: 'wiki list' },
});
return 0;
}
for (const page of pages) {
io.stdout.write(`${page.scope}\t${page.key}\t${page.summary}\n`);
}
@ -69,7 +78,15 @@ export async function runKtxKnowledge(
if (args.command === 'read') {
const page = await readLocalKnowledgePage(project, { key: args.key, userId: args.userId });
if (!page) {
throw new Error(`Knowledge page "${args.key}" was not found`);
throw new Error(`Wiki page "${args.key}" was not found`);
}
if (args.json) {
writeJsonResult(io, {
kind: 'wiki.page',
data: page,
meta: { command: 'wiki read' },
});
return 0;
}
io.stdout.write(`# ${page.key}\n\n`);
io.stdout.write(`Scope: ${page.scope}\n`);
@ -82,7 +99,16 @@ export async function runKtxKnowledge(
query: args.query,
userId: args.userId,
embeddingService: wikiSearchEmbeddingService(project, deps),
limit: args.limit,
});
if (args.json) {
writeJsonResult(io, {
kind: 'list',
data: { items: results },
meta: { command: 'wiki search' },
});
return 0;
}
if (results.length === 0) {
const pages = await listLocalKnowledgePages(project, { userId: args.userId });
if (pages.length === 0) {

2
packages/cli/src/local-adapters.ts
View file

@ -1,4 +1,3 @@
import { join } from 'node:path';
import {
createBigQueryLiveDatabaseIntrospection,
isKtxBigQueryConnectionConfig,
@ -298,7 +297,6 @@ function historicSqlOptionsForLocalRun(project: KtxLocalProject, options: KtxCli
const base = {
sqlAnalysis: ktxCliHistoricSqlAnalysis(options),
postgresBaselineRootDir: join(project.projectDir, '.ktx/cache/historic-sql'),
};
if (dialect === 'postgres') {

9
packages/cli/src/local-scan-connectors.test.ts
View file

@ -62,10 +62,7 @@ describe('createKtxCliScanConnector', () => {
expect(connector.driver).toBe('sqlite');
});
it.each([
['maxBytesBilled', ' maxBytesBilled: 123456789', 123456789],
['max_bytes_billed', ' max_bytes_billed: "987654321"', '987654321'],
])('passes BigQuery %s from standalone config', async (_label, byteCapLine, expectedMaxBytesBilled) => {
it('passes BigQuery max_bytes_billed from standalone config', async () => {
await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' });
await writeFile(
join(tempDir, 'ktx.yaml'),
@ -76,7 +73,7 @@ describe('createKtxCliScanConnector', () => {
' driver: bigquery',
' dataset_id: analytics',
' readonly: true',
byteCapLine,
' max_bytes_billed: "987654321"',
'',
].join('\n'),
'utf-8',
@ -90,7 +87,7 @@ describe('createKtxCliScanConnector', () => {
expect(bigQueryMock.constructorInputs).toEqual([
expect.objectContaining({
connectionId: 'warehouse',
maxBytesBilled: expectedMaxBytesBilled,
maxBytesBilled: '987654321',
}),
]);
});

2
packages/cli/src/local-scan-connectors.ts
View file

@ -6,7 +6,7 @@ const SUPPORTED_DRIVERS = 'sqlite, postgres, mysql, clickhouse, sqlserver, bigqu
function bigQueryMaxBytesBilled(
connection: KtxLocalProject['config']['connections'][string],
): number | string | undefined {
const raw = connection.maxBytesBilled ?? connection.max_bytes_billed;
const raw = connection.max_bytes_billed;
if (typeof raw === 'number') {
return Number.isFinite(raw) && raw > 0 ? raw : undefined;
}

41
packages/cli/src/managed-python-runtime.test.ts
View file

@ -1,5 +1,5 @@
import { createHash } from 'node:crypto';
import { mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from 'node:fs/promises';
import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
@ -8,7 +8,6 @@ import {
doctorManagedPythonRuntime,
installManagedPythonRuntime,
managedPythonRuntimeLayout,
pruneManagedPythonRuntimes,
readManagedPythonRuntimeStatus,
verifyRuntimeAsset,
type ManagedPythonRuntimeExec,
@ -471,41 +470,3 @@ describe('doctorManagedPythonRuntime', () => {
});
});
});
describe('pruneManagedPythonRuntimes', () => {
let tempDir: string;
beforeEach(async () => {
tempDir = await mkdtemp(join(tmpdir(), 'ktx-runtime-prune-'));
});
afterEach(async () => {
await rm(tempDir, { recursive: true, force: true });
});
it('removes stale version directories and keeps the current version', async () => {
const runtimeRoot = join(tempDir, 'runtime');
await mkdir(join(runtimeRoot, '0.1.0'), { recursive: true });
await mkdir(join(runtimeRoot, '0.2.0'), { recursive: true });
await writeFile(join(runtimeRoot, 'README.txt'), 'not a runtime directory\n');
const result = await pruneManagedPythonRuntimes({ cliVersion: '0.2.0', runtimeRoot });
expect(result.removed).toEqual([join(runtimeRoot, '0.1.0')]);
expect(result.kept).toEqual([join(runtimeRoot, '0.2.0')]);
await expect(stat(join(runtimeRoot, '0.1.0'))).rejects.toThrow();
expect(await readdir(runtimeRoot)).toEqual(['0.2.0', 'README.txt']);
});
it('supports dry-run without deleting stale directories', async () => {
const runtimeRoot = join(tempDir, 'runtime');
await mkdir(join(runtimeRoot, '0.1.0'), { recursive: true });
await mkdir(join(runtimeRoot, '0.2.0'), { recursive: true });
const result = await pruneManagedPythonRuntimes({ cliVersion: '0.2.0', runtimeRoot, dryRun: true });
expect(result.removed).toEqual([]);
expect(result.stale).toEqual([join(runtimeRoot, '0.1.0')]);
expect(await readdir(runtimeRoot)).toEqual(['0.1.0', '0.2.0']);
});
});

42
packages/cli/src/managed-python-runtime.ts
View file

@ -1,6 +1,6 @@
import { execFile } from 'node:child_process';
import { createHash } from 'node:crypto';
import { access, appendFile, mkdir, readFile, readdir, rm, stat, writeFile } from 'node:fs/promises';
import { access, appendFile, mkdir, readFile, rm, writeFile } from 'node:fs/promises';
import { homedir } from 'node:os';
import { basename, join } from 'node:path';
import { fileURLToPath } from 'node:url';
@ -107,13 +107,6 @@ export interface ManagedPythonRuntimeDoctorCheck {
fix?: string;
}
export interface ManagedPythonRuntimePruneResult {
runtimeRoot: string;
stale: string[];
kept: string[];
removed: string[];
}
export const MISSING_UV_RUNTIME_INSTALL_MESSAGE =
'uv is required to install the KTX Python runtime. KTX does not download uv automatically. Install uv, make sure it is on PATH, and retry: ktx dev runtime install --yes';
@ -441,36 +434,3 @@ export async function doctorManagedPythonRuntime(
);
return checks;
}
export async function pruneManagedPythonRuntimes(options: {
cliVersion: string;
runtimeRoot: string;
dryRun?: boolean;
}): Promise<ManagedPythonRuntimePruneResult> {
if (!(await pathExists(options.runtimeRoot))) {
return { runtimeRoot: options.runtimeRoot, stale: [], kept: [], removed: [] };
}
const entries = await readdir(options.runtimeRoot);
const stale: string[] = [];
const kept: string[] = [];
for (const entry of entries) {
const path = join(options.runtimeRoot, entry);
const info = await stat(path);
if (!info.isDirectory()) {
continue;
}
if (entry === options.cliVersion) {
kept.push(path);
} else {
stale.push(path);
}
}
const removed: string[] = [];
if (options.dryRun !== true) {
for (const path of stale) {
await rm(path, { recursive: true, force: true });
removed.push(path);
}
}
return { runtimeRoot: options.runtimeRoot, stale, kept, removed };
}

125
packages/cli/src/memory-flow-hud.tsx
View file

@ -1,7 +1,7 @@
/* @jsxImportSource react */
import type { MemoryFlowEvent, MemoryFlowReplayInput } from '@ktx/context/ingest/memory-flow';
import { Box, Text } from 'ink';
import React, { type ReactNode } from 'react';
import { type ReactNode } from 'react';
import { buildDemoMetrics, formatCost, formatDuration } from './demo-metrics.js';
import { formatNextStepLines } from './next-steps.js';
import { profileMark } from './startup-profile.js';
@ -38,45 +38,6 @@ function isPrepopulatedDemoReplay(input: MemoryFlowReplayInput): boolean {
return input.metadata?.origin === 'packaged' || input.metadata?.timing === 'prebuilt';
}
function flowLine(width: number, frame: number, active: boolean): string {
if (!active) return '━'.repeat(width);
const pulse = ['░', '▒', '▓', '█', '█', '█', '▓', '▒', '░'];
const pw = pulse.length;
const chars: string[] = [];
const offset = (frame * 2) % (width + pw);
for (let i = 0; i < width; i += 1) {
const p = i - offset + pw;
chars.push(p >= 0 && p < pw ? (pulse[p] ?? '━') : '━');
}
return chars.join('');
}
function brailleFlow(width: number, frame: number): string {
// Braille unicode: U+2800 + dot bitmask
// Dots: 1=0x01 2=0x02 3=0x04 4=0x08 5=0x10 6=0x20 7=0x40 8=0x80
// Layout: col0=[1,2,3,7] col1=[4,5,6,8]
const chars: string[] = [];
for (let i = 0; i < width; i += 1) {
const density = (i + 1) / width;
const phase = (i * 3 + frame * 2) % 12;
let dots = 0;
// Sparse diagonal streams on the left, dense on the right
// Each "stream" is a diagonal line of dots moving rightward
if ((phase + 0) % 4 < density * 4) dots |= 0x01; // dot 1
if ((phase + 1) % 5 < density * 4) dots |= 0x08; // dot 4
if ((phase + 2) % 4 < density * 3) dots |= 0x02; // dot 2
if ((phase + 3) % 5 < density * 3) dots |= 0x10; // dot 5
if ((phase + 4) % 4 < density * 2.5) dots |= 0x04; // dot 3
if ((phase + 5) % 5 < density * 2.5) dots |= 0x20; // dot 6
if ((phase + 1) % 6 < density * 2) dots |= 0x40; // dot 7
if ((phase + 3) % 6 < density * 2) dots |= 0x80; // dot 8
chars.push(String.fromCharCode(0x2800 + dots));
}
return chars.join('');
}
function progressBarOverall(
finishedCount: number,
activeCount: number,
@ -104,43 +65,6 @@ function progressBarOverall(
return finished + activeChars.join('') + '░'.repeat(queuedWidth);
}
function sparkleWipe(width: number, frame: number, row: number): string {
const chars: string[] = [];
const sweepPos = (frame * 2 + row * 6) % (width + 8);
const sparkles = ['✨', '✦', '✧', '·'];
for (let i = 0; i < width; i += 1) {
const dist = i - sweepPos;
if (dist < -6) {
const t = (i * 11 + row * 5 + frame * 3) % 10;
chars.push(t === 0 ? sparkles[0]! : t === 3 ? sparkles[1]! : t === 7 ? sparkles[2]! : ' ');
} else if (dist < -3) {
const t = (i + frame) % 3;
chars.push(t === 0 ? sparkles[1]! : t === 1 ? sparkles[2]! : sparkles[3]!);
} else if (dist <= 0) {
const gradient = ['░', '▒', '▓', '█'];
chars.push(gradient[Math.min(3, dist + 3)] ?? '█');
} else if (dist <= 2) {
chars.push(dist === 1 ? '▓' : '▒');
} else {
const noise = (i * 31 + row * 17 + frame * 3) % 5;
const messy = ['░', '▒', '▓', '▒', '░'];
chars.push(messy[noise] ?? '▒');
}
}
return chars.join('');
}
function activityWave(width: number, frame: number, offset: number): string {
const heights = ['▁', '▂', '▃', '▄', '▅', '▆', '▇', '█'];
const chars: string[] = [];
for (let i = 0; i < width; i += 1) {
const wave = Math.sin(((i * 2 + frame + offset * 5) * Math.PI) / 6);
const idx = Math.round(((wave + 1) / 2) * (heights.length - 1));
chars.push(heights[idx] ?? '▁');
}
return chars.join('');
}
function topicName(key: string): string {
return (key.split('/').pop()?.replace(/\.md$/, '') ?? key).replace(/[_-]/g, ' ');
}
@ -152,21 +76,12 @@ function tableName(key: string): string {
function humanizeInsight(key: string, target: 'sl' | 'wiki', summary: string | undefined): string {
if (summary) return summary;
const name = target === 'sl' ? tableName(key) : topicName(key);
return target === 'sl' ? `Query definition: ${name}` : `Knowledge page: ${name}`;
return target === 'sl' ? `Query definition: ${name}` : `Wiki page: ${name}`;
}
const ADAPTER_PREFIXES = ['live_database_', 'metabase_', 'looker_', 'lookml_', 'metricflow_', 'notion_', 'historic_sql_', 'dbt_descriptions_'];
const INTERNAL_DEMO_CONNECTION_ID = 'orbit_demo';
const PUBLIC_DEMO_SOURCE_LABEL = 'Orbit Demo';
function humanizeUnitKey(unitKey: string): string {
let key = unitKey.replace(/-/g, '_');
for (const prefix of ADAPTER_PREFIXES) {
if (key.startsWith(prefix)) { key = key.slice(prefix.length); break; }
}
return key.replace(/_/g, ' ');
}
interface SourceInfo {
type: string;
name: string;
@ -224,13 +139,6 @@ function sourceDescription(input: MemoryFlowReplayInput): SourceInfo {
return { type: info.type, name: conn, sourceCount: count, itemNounPlural: info.plural, readingVerb: info.verb, ingestDescription: info.description };
}
function activeWorkUnit(
input: MemoryFlowReplayInput,
): { unitKey: string; stepIndex: number; stepBudget: number } | null {
const units = activeWorkUnits(input);
return units.at(-1) ?? null;
}
function activeWorkUnits(
input: MemoryFlowReplayInput,
): Array<{ unitKey: string; stepIndex: number; stepBudget: number }> {
@ -299,22 +207,6 @@ function finishedUnits(input: MemoryFlowReplayInput): Array<{ unitKey: string; a
return units;
}
function artifactCounts(input: MemoryFlowReplayInput): { sl: number; wiki: number } {
let sl = 0;
let wiki = 0;
for (const e of input.events) {
if (e.type === 'candidate_action') {
if (e.target === 'sl') sl++;
else wiki++;
}
}
return { sl, wiki };
}
function pad(str: string, width: number): string {
return str.length >= width ? str : str + ' '.repeat(width - str.length);
}
const KTX_LOGO_SMALL = [
'██╗ ██╗████████╗██╗ ██╗',
'██║ ██╔╝╚══██╔══╝╚██╗██╔╝',
@ -344,12 +236,7 @@ export function Hud(props: {
width: number;
now?: () => number;
}): ReactNode {
const isRunning = props.input.status === 'running';
const isDone = props.input.status === 'done';
const isFlowing = isRunning && hasWorkStarted(props.input);
const src = sourceDescription(props.input);
const counts = artifactCounts(props.input);
const metrics = buildDemoMetrics(props.input, props.now ? { now: props.now } : {});
const workStarted = hasWorkStarted(props.input);
@ -358,11 +245,6 @@ export function Hud(props: {
const innerWidth = Math.max(60, props.width - 6);
const actives = activeWorkUnits(props.input);
const reconEvent = props.input.events.find((e) => e.type === 'reconciliation_finished');
const allAnalyzed = isFlowing && actives.length === 0;
const isReconciling = allAnalyzed && !reconEvent && !isDone;
const hLine = '─'.repeat(innerWidth);
const elapsed = formatDuration(metrics.elapsedMs);
@ -429,7 +311,6 @@ export function ActivityFeed(props: {
const workStarted = hasWorkStarted(props.input);
const totalChunks = planEvent?.chunkCount ?? 0;
const finishedWithArtifacts = finished.filter((u) => u.artifactCount > 0);
const finishedAreas = totalChunks > 0 ? Math.min(finished.length, totalChunks) : finished.length;
const allWorkDone = workStarted && actives.length === 0 && queued.length === 0;
const isReconciling = allWorkDone && !reconEvent && !isDone && !isError;
@ -572,7 +453,7 @@ function CompletionSummary(props: {
)}
{wiki > 0 && (
<Text color={props.theme.complete}>
{' '}📝 {wiki} knowledge page{wiki === 1 ? '' : 's'} — so agents understand your business context
{' '}📝 {wiki} wiki page{wiki === 1 ? '' : 's'} — so agents understand your business context
</Text>
)}
</>

4
packages/cli/src/memory-flow-interactive.test.ts
View file

@ -46,9 +46,9 @@ function replay(): MemoryFlowReplayInput {
{ type: 'raw_snapshot_written', syncId: 'sync-1', rawFileCount: 2 },
{ type: 'diff_computed', added: 1, modified: 1, deleted: 0, unchanged: 0 },
{ type: 'chunks_planned', chunkCount: 2, workUnitCount: 2, evictionCount: 0 },
{ type: 'work_unit_started', unitKey: 'orders', skills: ['knowledge_capture'], stepBudget: 4 },
{ type: 'work_unit_started', unitKey: 'orders', skills: ['wiki_capture'], stepBudget: 4 },
{ type: 'work_unit_finished', unitKey: 'orders', status: 'success' },
{ type: 'work_unit_started', unitKey: 'customers', skills: ['knowledge_capture'], stepBudget: 4 },
{ type: 'work_unit_started', unitKey: 'customers', skills: ['wiki_capture'], stepBudget: 4 },
{ type: 'work_unit_finished', unitKey: 'customers', status: 'failed', reason: 'validation reset' },
{ type: 'reconciliation_finished', conflictCount: 0, fallbackCount: 1 },
{ type: 'saved', commitSha: 'abc12345', wikiCount: 1, slCount: 1 },

13
packages/cli/src/memory-flow-tui.test.tsx
View file

@ -11,7 +11,6 @@ import {
startLiveMemoryFlowTui,
type KtxMemoryFlowTuiIo,
type MemoryFlowInkInstance,
type MemoryFlowInkRenderOptions,
} from './memory-flow-tui.js';
function replayInput(): MemoryFlowReplayInput {
@ -24,10 +23,10 @@ function replayInput(): MemoryFlowReplayInput {
],
details: {
actions: [
{ unitKey: 'orders', target: 'wiki', action: 'created', key: 'knowledge/orders.md', summary: 'order lifecycle', rawFiles: ['orders'], status: 'success' },
{ unitKey: 'orders', target: 'wiki', action: 'created', key: 'wiki/orders.md', summary: 'order lifecycle', rawFiles: ['orders'], status: 'success' },
{ unitKey: 'customers', target: 'sl', action: 'updated', key: 'orbit_demo.customers', summary: 'customer metrics', rawFiles: ['customers'], status: 'success' },
],
provenance: [{ rawPath: 'orders', artifactKind: 'wiki', artifactKey: 'knowledge/orders.md', actionType: 'wiki_written' }],
provenance: [{ rawPath: 'orders', artifactKind: 'wiki', artifactKey: 'wiki/orders.md', actionType: 'wiki_written' }],
transcripts: [{ unitKey: 'orders', path: '/tmp/t.jsonl', toolCallCount: 2, errorCount: 0, toolNames: ['read_raw_span', 'wiki_write'] }],
},
events: [
@ -36,8 +35,8 @@ function replayInput(): MemoryFlowReplayInput {
{ type: 'raw_snapshot_written', syncId: 'sync-1', rawFileCount: 2 },
{ type: 'diff_computed', added: 1, modified: 1, deleted: 0, unchanged: 0 },
{ type: 'chunks_planned', chunkCount: 2, workUnitCount: 2, evictionCount: 0 },
{ type: 'work_unit_started', unitKey: 'orders', skills: ['knowledge_capture'], stepBudget: 40 },
{ type: 'candidate_action', unitKey: 'orders', target: 'wiki', action: 'created', key: 'knowledge/orders.md' },
{ type: 'work_unit_started', unitKey: 'orders', skills: ['wiki_capture'], stepBudget: 40 },
{ type: 'candidate_action', unitKey: 'orders', target: 'wiki', action: 'created', key: 'wiki/orders.md' },
{ type: 'work_unit_finished', unitKey: 'orders', status: 'success' },
{ type: 'work_unit_started', unitKey: 'customers', skills: ['sl_capture'], stepBudget: 40 },
{ type: 'candidate_action', unitKey: 'customers', target: 'sl', action: 'updated', key: 'orbit_demo.customers' },
@ -221,7 +220,7 @@ describe('MemoryFlowTuiApp', () => {
{ type: 'source_acquired', adapter: 'live-database', trigger: 'manual_resync', fileCount: 1 },
{ type: 'diff_computed', added: 1, modified: 0, deleted: 0, unchanged: 0 },
{ type: 'chunks_planned', chunkCount: 1, workUnitCount: 1, evictionCount: 0 },
{ type: 'work_unit_started', unitKey: 'orders', skills: ['knowledge_capture'], stepBudget: 40 },
{ type: 'work_unit_started', unitKey: 'orders', skills: ['wiki_capture'], stepBudget: 40 },
],
plannedWorkUnits: [{ unitKey: 'orders', rawFiles: ['orders'], peerFileCount: 0, dependencyCount: 1 }],
};
@ -241,7 +240,7 @@ describe('MemoryFlowTuiApp', () => {
{ type: 'source_acquired', adapter: 'dbt-descriptions', trigger: 'manual_resync', fileCount: 3 },
{ type: 'diff_computed', added: 11, modified: 0, deleted: 0, unchanged: 0 },
{ type: 'chunks_planned', chunkCount: 1, workUnitCount: 1, evictionCount: 0 },
{ type: 'work_unit_started', unitKey: 'orders', skills: ['knowledge_capture'], stepBudget: 40 },
{ type: 'work_unit_started', unitKey: 'orders', skills: ['wiki_capture'], stepBudget: 40 },
],
plannedWorkUnits: [{ unitKey: 'orders', rawFiles: ['orders'], peerFileCount: 0, dependencyCount: 1 }],
};

19
packages/cli/src/memory-flow-tui.tsx
View file

@ -1,7 +1,6 @@
/* @jsxImportSource react */
import {
buildMemoryFlowViewModel,
buildMemoryFlowVisualModel,
createInitialMemoryFlowInteractionState,
findMemoryFlowSearchMatches,
type MemoryFlowColumnId,
@ -14,8 +13,7 @@ import {
selectedMemoryFlowDetails,
} from '@ktx/context/ingest';
import { Box, Text, render as renderInkRuntime, useApp, useInput } from 'ink';
import React, { type ReactNode, useEffect, useMemo, useRef, useState } from 'react';
import { buildDemoMetrics } from './demo-metrics.js';
import { type ReactNode, useEffect, useMemo, useRef, useState } from 'react';
import {
ActivityFeed,
Hud,
@ -201,14 +199,6 @@ function stageLabel(columnId: MemoryFlowColumnId): string {
return STAGE_LABELS[columnId];
}
function statusLabel(status: string): 'OK' | 'RUN' | 'WARN' | 'FAIL' | 'WAIT' {
if (status === 'complete') return 'OK';
if (status === 'active') return 'RUN';
if (status === 'warning') return 'WARN';
if (status === 'failed') return 'FAIL';
return 'WAIT';
}
function filterLabel(filter: MemoryFlowInteractionState['filter']): string {
return filter === 'failed_or_flagged' ? 'issues' : 'all';
}
@ -325,7 +315,6 @@ export function MemoryFlowTuiApp(props: MemoryFlowTuiAppProps): ReactNode {
const view = useMemo(() => buildMemoryFlowViewModel(pacedInput), [pacedInput]);
const [state, setState] = useState<MemoryFlowInteractionState>(() => createInitialMemoryFlowInteractionState(view));
const [frame, setFrame] = useState(0);
const [thoughtFrame, setThoughtFrame] = useState(0);
const [completionFrame, setCompletionFrame] = useState(0);
const [holdComplete, setHoldComplete] = useState(false);
const [userHasNavigated, setUserHasNavigated] = useState(false);
@ -346,7 +335,6 @@ export function MemoryFlowTuiApp(props: MemoryFlowTuiAppProps): ReactNode {
useEffect(() => {
const timer = setInterval(() => {
setFrame((current) => current + 1);
setThoughtFrame((current) => current + 1);
}, props.frameMs ?? DEFAULT_TUI_TIMING.frameMs);
return () => clearInterval(timer);
}, [props.frameMs]);
@ -354,7 +342,6 @@ export function MemoryFlowTuiApp(props: MemoryFlowTuiAppProps): ReactNode {
useEffect(() => {
if (lastEventCountRef.current !== pacedInput.events.length) {
lastEventCountRef.current = pacedInput.events.length;
setThoughtFrame(0);
}
}, [pacedInput.events.length]);
@ -409,10 +396,6 @@ export function MemoryFlowTuiApp(props: MemoryFlowTuiAppProps): ReactNode {
});
const isComplete = pacedInput.status === 'done' || pacedInput.status === 'error';
const completionMetrics = useMemo(
() => buildDemoMetrics(pacedInput, pacedNow ? { now: pacedNow } : {}),
[pacedInput, pacedNow],
);
const termWidth = props.terminalWidth ?? 80;

40
packages/cli/src/next-steps.test.ts
View file

@ -6,8 +6,6 @@ import {
formatSetupNextStepLines,
} from './next-steps.js';
const command = (...parts: string[]) => parts.join(' ');
describe('KTX demo next steps', () => {
it('uses supported context-build commands before agent usage', () => {
expect(KTX_CONTEXT_BUILD_COMMANDS).toEqual([
@ -25,12 +23,8 @@ describe('KTX demo next steps', () => {
it('uses supported final public commands', () => {
expect(KTX_NEXT_STEP_COMMANDS).toEqual([
{
command: 'ktx agent context --json',
description: 'Verify the project context your agent can read',
},
{
command: 'ktx agent tools --json',
description: 'List direct CLI tools available to agents',
command: 'ktx status --json',
description: 'Verify project setup and context readiness',
},
{
command: 'ktx sl list',
@ -46,8 +40,8 @@ describe('KTX demo next steps', () => {
it('uses only the direct CLI route for agent verification', () => {
const commands = KTX_NEXT_STEP_COMMANDS.map((step) => step.command);
expect(commands).toContain('ktx agent context --json');
expect(commands).toContain('ktx agent tools --json');
expect(commands).not.toContain('ktx agent context --json');
expect(commands).toContain('ktx status --json');
expect(commands).not.toContain('ktx serve --mcp stdio --user-id local');
});
@ -61,29 +55,6 @@ describe('KTX demo next steps', () => {
expect(rendered).not.toContain('Optional MCP:');
});
it('does not advertise removed Commander migration commands', () => {
const rendered = formatNextStepLines().join('\n');
expect(rendered).toContain('ktx agent tools --json');
expect(rendered).toContain('ktx agent context --json');
expect(rendered).toContain('ktx sl list');
expect(rendered).toContain('ktx wiki list');
for (const removed of [
command('ktx', 'ask'),
command('ktx', 'mcp'),
command('ktx', 'connect'),
command('ktx', 'knowledge'),
command('dev', 'model'),
command('dev', 'knowledge'),
command('ktx', 'ingest', 'run'),
command('ktx', 'ingest', 'replay'),
command('ktx', 'serve', '--mcp', 'stdio', '--user-id', 'local'),
]) {
expect(rendered).not.toContain(removed);
}
});
it('keeps setup next steps focused on building context when the build is not ready', () => {
const rendered = formatSetupNextStepLines({
setupReady: true,
@ -109,7 +80,8 @@ describe('KTX demo next steps', () => {
}).join('\n');
expect(rendered).toContain('KTX context is ready for agents.');
expect(rendered).toContain('ktx agent context --json');
expect(rendered).toContain('ktx status --json');
expect(rendered).not.toContain('ktx agent');
expect(rendered).not.toContain('ktx serve --mcp stdio --user-id local');
expect(rendered).not.toContain('Build KTX context next.');
});

8
packages/cli/src/next-steps.ts
View file

@ -11,12 +11,8 @@ export const KTX_CONTEXT_BUILD_COMMANDS = [
export const KTX_NEXT_STEP_DIRECT_COMMANDS = [
{
command: 'ktx agent context --json',
description: 'Verify the project context your agent can read',
},
{
command: 'ktx agent tools --json',
description: 'List direct CLI tools available to agents',
command: 'ktx status --json',
description: 'Verify project setup and context readiness',
},
{
command: 'ktx sl list',

2
packages/cli/src/commands/connection-notion-tree.test.ts → packages/cli/src/notion-page-picker-tree.test.ts
View file

@ -14,7 +14,7 @@ import {
TRANSIENT_HINT_DURATION_MS,
visibleNodeIds,
type NotionPickerPageInput,
} from './connection-notion-tree.js';
} from './notion-page-picker-tree.js';
const IDS = {
engineering: '11111111-1111-1111-1111-111111111111',

8
packages/cli/src/commands/connection-notion-tree.ts → packages/cli/src/notion-page-picker-tree.ts
View file

@ -369,14 +369,6 @@ function setExpanded(state: PickerState, nodeId: string, value: boolean | 'toggl
return cloneState(state, { expanded });
}
function expandPath(state: PickerState, nodeId: string): PickerState {
const expanded = new Set(state.expanded);
for (const ancestorId of ancestorsOf(nodeId, state.byId)) {
expanded.add(ancestorId);
}
return cloneState(state, { expanded });
}
export function moveCursor(state: PickerState, dir: 'up' | 'down' | 'left' | 'right'): PickerState {
const node = state.byId.get(state.cursorId);
if (!node) {

8
packages/cli/src/commands/connection-notion-tui.test.tsx → packages/cli/src/notion-page-picker-tui.test.tsx
View file

@ -1,8 +1,8 @@
/* @jsxImportSource react */
import { render as renderInkTest } from 'ink-testing-library';
import React, { act, type ReactNode } from 'react';
import { act, type ReactNode } from 'react';
import { afterEach, describe, expect, it, vi } from 'vitest';
import { buildInitialState, buildPickerTree, type NotionPickerPageInput } from './connection-notion-tree.js';
import { buildInitialState, buildPickerTree, type NotionPickerPageInput } from './notion-page-picker-tree.js';
import {
NotionPickerApp,
notionPickerCommandForInkInput,
@ -13,7 +13,7 @@ import {
windowOffset,
type NotionPickerInkInstance,
type NotionPickerInkRenderOptions,
} from './connection-notion-tui.js';
} from './notion-page-picker-tui.js';
const IDS = {
engineering: '11111111-1111-1111-1111-111111111111',
@ -378,7 +378,7 @@ describe('renderNotionPickerTui', () => {
},
),
).resolves.toEqual({ kind: 'quit' });
expect(stderr).toContain('Use --no-input --root-page-id <UUID> for scripted mode');
expect(stderr).toContain('Use --no-input --notion-root-page-id <UUID> for scripted mode');
expect(stderr).not.toContain('secret');
});
});

8
packages/cli/src/commands/connection-notion-tui.tsx → packages/cli/src/notion-page-picker-tui.tsx
View file

@ -1,6 +1,6 @@
/* @jsxImportSource react */
import { Box, Text, render as renderInkRuntime, useApp, useInput } from 'ink';
import React, { type ReactNode, useEffect, useMemo, useRef, useState } from 'react';
import { type ReactNode, useEffect, useMemo, useRef, useState } from 'react';
import {
filterTree,
flattenSelection,
@ -9,8 +9,8 @@ import {
visibleNodeIds,
type PickerCommand,
type PickerState,
} from './connection-notion-tree.js';
import type { KtxCliIo } from '../index.js';
} from './notion-page-picker-tree.js';
import type { KtxCliIo } from './cli-runtime.js';
const COLOR_THEME = {
text: 'white',
@ -331,7 +331,7 @@ export async function renderNotionPickerTui(
return result;
} catch (error) {
io.stderr.write(
`Notion picker requires a TTY. Use --no-input --root-page-id <UUID> for scripted mode. ${sanitizeNotionPickerTuiError(error)}\n`,
`Notion picker requires a TTY. Use --no-input --notion-root-page-id <UUID> for scripted mode. ${sanitizeNotionPickerTuiError(error)}\n`,
);
return { kind: 'quit' };
}

308
packages/cli/src/notion-page-picker.test.ts Normal file
View file

@ -0,0 +1,308 @@
import { describe, expect, it, vi } from 'vitest';
import {
discoverNotionPickerPages,
notionPickerPageFromSearchResult,
normalizeNotionPageId,
pickNotionRootPages,
resolveNotionWorkspaceLabel,
type NotionPickerApi,
type PickerRenderInput,
type PickerRenderResult,
} from './notion-page-picker.js';
function makeIo() {
let stdout = '';
let stderr = '';
return {
io: {
stdout: {
isTTY: true,
write: (chunk: string) => {
stdout += chunk;
},
},
stderr: {
write: (chunk: string) => {
stderr += chunk;
},
},
},
stdout: () => stdout,
stderr: () => stderr,
};
}
type FakeNotionSearchPage = Record<string, unknown> & { id: string; object: 'page' };
const PAGE_IDS = {
engineering: '11111111-1111-1111-1111-111111111111',
architecture: '22222222-2222-2222-2222-222222222222',
stale: '99999999-9999-9999-9999-999999999999',
};
function notionPage(id: string, title: string, parentId: string | null = null): FakeNotionSearchPage {
return {
object: 'page',
id,
archived: false,
parent: parentId ? { type: 'page_id', page_id: parentId } : { type: 'workspace', workspace: true },
properties: {
title: {
type: 'title',
title: [{ plain_text: title }],
},
},
};
}
function fakeNotionApi(pages: FakeNotionSearchPage[]): NotionPickerApi {
return {
search: vi.fn(async (_filterValue, startCursor) => {
if (startCursor === 'page-2') {
return { results: pages.slice(2), hasMore: false, nextCursor: null };
}
return {
results: pages.slice(0, 2),
hasMore: pages.length > 2,
nextCursor: pages.length > 2 ? 'page-2' : null,
};
}),
retrieveBotUser: vi.fn(async () => ({ name: 'Notion bot', bot: { workspace_name: 'Design Workspace' } })),
};
}
describe('normalizeNotionPageId', () => {
it('accepts dashed and compact UUIDs', () => {
expect(normalizeNotionPageId('11111111222233334444555555555555')).toBe(
'11111111-2222-3333-4444-555555555555',
);
expect(normalizeNotionPageId('AAAAAAAA-BBBB-CCCC-DDDD-EEEEEEEEEEEE')).toBe(
'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee',
);
});
});
describe('Notion page picker helpers', () => {
it('extracts picker page inputs from Notion search results', () => {
expect(notionPickerPageFromSearchResult(notionPage(PAGE_IDS.architecture, 'Architecture', PAGE_IDS.engineering)))
.toEqual({
id: PAGE_IDS.architecture,
title: 'Architecture',
archived: false,
parentId: PAGE_IDS.engineering,
});
expect(
notionPickerPageFromSearchResult({
object: 'page',
id: PAGE_IDS.engineering.replaceAll('-', ''),
archived: true,
parent: { type: 'workspace', workspace: true },
properties: {},
}),
).toEqual({
id: PAGE_IDS.engineering,
title: 'Untitled',
archived: true,
parentId: null,
});
});
it('discovers visible pages up to the cap and reports cap state', async () => {
const api = fakeNotionApi([
notionPage(PAGE_IDS.engineering, 'Engineering'),
notionPage(PAGE_IDS.architecture, 'Architecture', PAGE_IDS.engineering),
notionPage('33333333-3333-3333-3333-333333333333', 'Onboarding', PAGE_IDS.engineering),
]);
await expect(discoverNotionPickerPages(api, { cap: 2 })).resolves.toEqual({
pages: [
{ id: PAGE_IDS.engineering, title: 'Engineering', archived: false, parentId: null },
{ id: PAGE_IDS.architecture, title: 'Architecture', archived: false, parentId: PAGE_IDS.engineering },
],
cappedAtCount: 2,
warnings: [],
});
expect(api.search).toHaveBeenCalledTimes(1);
});
it('keeps partial discovery results when Notion search fails after at least one page', async () => {
const api: NotionPickerApi = {
search: vi
.fn()
.mockResolvedValueOnce({
results: [notionPage(PAGE_IDS.engineering, 'Engineering')],
hasMore: true,
nextCursor: 'cursor-2',
})
.mockRejectedValueOnce(new Error('rate limit after first page')),
retrieveBotUser: vi.fn(async () => ({ name: 'Notion bot' })),
};
await expect(discoverNotionPickerPages(api)).resolves.toEqual({
pages: [{ id: PAGE_IDS.engineering, title: 'Engineering', archived: false, parentId: null }],
cappedAtCount: null,
warnings: ['Notion search stopped early: rate limit after first page'],
});
});
it('uses the Notion workspace name when available and falls back to the connection id', async () => {
await expect(resolveNotionWorkspaceLabel(fakeNotionApi([]), 'notion-main')).resolves.toBe('Design Workspace');
await expect(
resolveNotionWorkspaceLabel(
{
search: vi.fn(),
retrieveBotUser: vi.fn(async () => {
throw new Error('users.me unavailable');
}),
},
'notion-main',
),
).resolves.toBe('notion-main');
});
});
describe('pickNotionRootPages', () => {
it('discovers visible pages, warns about stale roots, renders the TUI, and returns selected roots', async () => {
const api = fakeNotionApi([
notionPage(PAGE_IDS.engineering, 'Engineering'),
notionPage(PAGE_IDS.architecture, 'Architecture', PAGE_IDS.engineering),
]);
const renderPicker = vi.fn(async (input: PickerRenderInput): Promise<PickerRenderResult> => {
expect(input.connectionId).toBe('notion-main');
expect(input.workspaceLabel).toBe('Design Workspace');
expect(input.currentCrawlMode).toBe('all_accessible');
expect(input.cappedAtCount).toBeNull();
expect(input.initialState.preLoadWarnings).toEqual(['1 stored root_page_ids no longer visible']);
return { kind: 'save', rootPageIds: [PAGE_IDS.engineering] };
});
const io = makeIo();
await expect(
pickNotionRootPages(
{
connectionId: 'notion-main',
connection: {
driver: 'notion',
auth_token_ref: 'env:NOTION_TOKEN',
crawl_mode: 'all_accessible',
root_page_ids: [PAGE_IDS.stale],
},
},
io.io,
{
env: { NOTION_TOKEN: 'ntn_test_token' },
createNotionApi: vi.fn(() => api),
renderPicker,
},
),
).resolves.toEqual({ kind: 'selected', rootPageIds: [PAGE_IDS.engineering] });
expect(io.stderr()).toContain('1 stored root_page_ids no longer visible');
expect(io.stdout()).toBe('');
});
it('uses inline Notion auth_token for discovery', async () => {
const api = fakeNotionApi([notionPage(PAGE_IDS.engineering, 'Engineering')]);
const createNotionApi = vi.fn((authToken: string) => {
expect(authToken).toBe('ntn_inline_token');
return api;
});
await expect(
pickNotionRootPages(
{
connectionId: 'notion-main',
connection: {
driver: 'notion',
auth_token: 'ntn_inline_token',
crawl_mode: 'selected_roots',
root_page_ids: [PAGE_IDS.engineering],
},
},
makeIo().io,
{
createNotionApi,
renderPicker: vi.fn(async (): Promise<PickerRenderResult> => ({ kind: 'quit' })),
},
),
).resolves.toEqual({ kind: 'back' });
expect(createNotionApi).toHaveBeenCalledOnce();
});
it('passes partial-discovery warnings into the TUI banner state', async () => {
const api: NotionPickerApi = {
search: vi
.fn()
.mockResolvedValueOnce({
results: [notionPage(PAGE_IDS.engineering, 'Engineering')],
hasMore: true,
nextCursor: 'cursor-2',
})
.mockRejectedValueOnce(new Error('rate limit after first page')),
retrieveBotUser: vi.fn(async () => ({ name: 'Notion bot', bot: { workspace_name: 'Design Workspace' } })),
};
let renderInput: PickerRenderInput | undefined;
const renderPicker = vi.fn(async (input: PickerRenderInput): Promise<PickerRenderResult> => {
renderInput = input;
return { kind: 'quit' };
});
const io = makeIo();
await expect(
pickNotionRootPages(
{
connectionId: 'notion-main',
connection: {
driver: 'notion',
auth_token_ref: 'env:NOTION_TOKEN',
crawl_mode: 'selected_roots',
root_page_ids: [PAGE_IDS.engineering],
},
},
io.io,
{
env: { NOTION_TOKEN: 'ntn_test_token' },
createNotionApi: vi.fn(() => api),
renderPicker,
},
),
).resolves.toEqual({ kind: 'back' });
expect(renderPicker).toHaveBeenCalledOnce();
if (!renderInput) {
throw new Error('renderPicker was not called');
}
expect(renderInput.initialState.preLoadWarnings).toEqual(['Notion search stopped early: rate limit after first page']);
expect(renderInput.initialState.tree.map((node) => node.title)).toEqual(['Engineering']);
expect(io.stderr()).toContain('Notion search stopped early: rate limit after first page');
});
it('returns unavailable when discovery cannot load any pages', async () => {
await expect(
pickNotionRootPages(
{
connectionId: 'notion-main',
connection: {
driver: 'notion',
auth_token_ref: 'env:NOTION_TOKEN',
crawl_mode: 'selected_roots',
root_page_ids: [],
},
},
makeIo().io,
{
env: { NOTION_TOKEN: 'ntn_test_token' },
createNotionApi: vi.fn(() => ({
search: vi.fn(async () => {
throw new Error('Notion API unavailable');
}),
retrieveBotUser: vi.fn(async () => ({ name: 'Notion bot' })),
})),
renderPicker: vi.fn(async (): Promise<PickerRenderResult> => ({ kind: 'quit' })),
},
),
).resolves.toEqual({ kind: 'unavailable', message: 'Notion API unavailable' });
});
});

200
packages/cli/src/commands/connection-notion.ts → packages/cli/src/notion-page-picker.ts
View file

@ -1,51 +1,40 @@
import { parseNotionConnectionConfig, resolveNotionConnectionAuthToken } from '@ktx/context/connections';
import { resolveNotionConnectionAuthToken } from '@ktx/context/connections';
import { type NotionApi, type NotionBotInfo, NotionClient } from '@ktx/context/ingest';
import {
type KtxLocalProject,
type KtxProjectConnectionConfig,
loadKtxProject,
serializeKtxProjectConfig,
} from '@ktx/context/project';
import type { KtxCliIo } from '../index.js';
import { profileMark } from '../startup-profile.js';
import { buildInitialState, buildPickerTree, type NotionPickerPageInput } from './connection-notion-tree.js';
import type { KtxProjectConnectionConfig } from '@ktx/context/project';
import type { KtxCliIo } from './cli-runtime.js';
import { profileMark } from './startup-profile.js';
import { buildInitialState, buildPickerTree, type NotionPickerPageInput } from './notion-page-picker-tree.js';
import {
type NotionPickerTuiIo,
type PickerRenderInput,
type PickerRenderResult,
renderNotionPickerTui,
} from './connection-notion-tui.js';
} from './notion-page-picker-tui.js';
profileMark('module:commands/connection-notion');
profileMark('module:notion-page-picker');
export type KtxConnectionNotionArgs =
| {
command: 'pick';
projectDir: string;
connectionId: string;
mode: 'interactive';
}
| {
command: 'pick';
projectDir: string;
connectionId: string;
mode: 'non-interactive';
rootPageIds: string[];
};
export interface PickNotionRootPagesArgs {
connectionId: string;
connection: KtxProjectConnectionConfig;
}
export type NotionPickerApi = Pick<NotionApi, 'search' | 'retrieveBotUser'>;
export type { PickerRenderInput, PickerRenderResult };
interface KtxConnectionNotionDeps {
export type NotionRootPagePickResult =
| { kind: 'selected'; rootPageIds: string[] }
| { kind: 'back' }
| { kind: 'unavailable'; message: string };
export interface NotionRootPagePickerDeps {
env?: Record<string, string | undefined>;
loadProject?: typeof loadKtxProject;
createNotionApi?: (authToken: string) => NotionPickerApi;
renderPicker?: (input: PickerRenderInput, io: NotionPickerTuiIo) => Promise<PickerRenderResult>;
}
const NOTION_PICKER_PAGE_CAP = 5000;
function assertSafeConnectionId(connectionId: string): void {
function assertSafeNotionPickerConnectionId(connectionId: string): void {
if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(connectionId)) {
throw new Error(`Unsafe connection id: ${connectionId}`);
}
@ -168,111 +157,74 @@ export async function resolveNotionWorkspaceLabel(api: NotionPickerApi, connecti
}
}
function notionConnection(project: KtxLocalProject, connectionId: string): KtxProjectConnectionConfig {
const connection = project.config.connections[connectionId];
if (!connection) {
throw new Error(`Connection "${connectionId}" not found`);
}
function assertNotionConnection(connection: KtxProjectConnectionConfig, connectionId: string): void {
if (connection.driver !== 'notion') {
throw new Error(`Connection "${connectionId}" is not a Notion connection`);
}
return connection;
}
export async function applyNotionPickerWriteback(
project: KtxLocalProject,
connectionId: string,
rootPageIds: string[],
): Promise<void> {
if (rootPageIds.length === 0) {
throw new Error('connection notion pick requires at least one root page id');
}
const existing = notionConnection(project, connectionId);
const nextConfig = {
...project.config,
connections: {
...project.config.connections,
[connectionId]: {
...existing,
crawl_mode: 'selected_roots',
root_page_ids: rootPageIds,
},
},
};
await project.fileStore.writeFile(
'ktx.yaml',
serializeKtxProjectConfig(nextConfig),
'ktx',
'ktx@example.com',
`Pick Notion roots: ${connectionId} (${rootPageIds.length} pages)`,
);
function stringArray(value: unknown): string[] {
return Array.isArray(value) ? value.filter((item): item is string => typeof item === 'string' && item.trim().length > 0) : [];
}
export async function runKtxConnectionNotion(
args: KtxConnectionNotionArgs,
function notionCrawlMode(connection: KtxProjectConnectionConfig): 'all_accessible' | 'selected_roots' {
return connection.crawl_mode === 'all_accessible' ? 'all_accessible' : 'selected_roots';
}
export async function pickNotionRootPages(
args: PickNotionRootPagesArgs,
io: KtxCliIo = process,
deps: KtxConnectionNotionDeps = {},
): Promise<number> {
deps: NotionRootPagePickerDeps = {},
): Promise<NotionRootPagePickResult> {
try {
assertSafeConnectionId(args.connectionId);
const loadProject = deps.loadProject ?? loadKtxProject;
if (args.mode === 'interactive') {
const project = await loadProject({ projectDir: args.projectDir });
const rawConnection = notionConnection(project, args.connectionId);
const notion = parseNotionConnectionConfig(rawConnection);
const authToken = await resolveNotionConnectionAuthToken(notion, { env: deps.env });
const api = deps.createNotionApi ? deps.createNotionApi(authToken) : new NotionClient(authToken);
const discovery = await discoverNotionPickerPages(api);
const tree = buildPickerTree(discovery.pages);
const initialState = buildInitialState({
tree,
existingRootPageIds: notion.root_page_ids,
currentCrawlMode: notion.crawl_mode,
});
const preLoadWarnings = [...discovery.warnings, ...initialState.preLoadWarnings];
const renderState =
preLoadWarnings.length > 0
? {
...initialState,
preLoadWarnings,
}
: initialState;
for (const warning of preLoadWarnings) {
io.stderr.write(`${warning}\n`);
}
const workspaceLabel = await resolveNotionWorkspaceLabel(api, args.connectionId);
const result = await (deps.renderPicker ?? renderNotionPickerTui)(
{
initialState: renderState,
connectionId: args.connectionId,
workspaceLabel,
cappedAtCount: discovery.cappedAtCount,
currentCrawlMode: notion.crawl_mode,
},
io as NotionPickerTuiIo,
);
if (result.kind === 'quit') {
io.stdout.write('No changes saved.\n');
return 0;
}
await applyNotionPickerWriteback(project, args.connectionId, result.rootPageIds);
io.stdout.write(`Connection: ${args.connectionId}\n`);
io.stdout.write(`rootPageIds: ${result.rootPageIds.length}\n`);
io.stdout.write('crawlMode: selected_roots\n');
return 0;
assertSafeNotionPickerConnectionId(args.connectionId);
assertNotionConnection(args.connection, args.connectionId);
const crawlMode = notionCrawlMode(args.connection);
const authToken = await resolveNotionConnectionAuthToken(
{
auth_token: typeof args.connection.auth_token === 'string' ? args.connection.auth_token : null,
auth_token_ref: typeof args.connection.auth_token_ref === 'string' ? args.connection.auth_token_ref : null,
},
{ env: deps.env },
);
const api = deps.createNotionApi ? deps.createNotionApi(authToken) : new NotionClient(authToken);
const discovery = await discoverNotionPickerPages(api);
const tree = buildPickerTree(discovery.pages);
const initialState = buildInitialState({
tree,
existingRootPageIds: stringArray(args.connection.root_page_ids),
currentCrawlMode: crawlMode,
});
const preLoadWarnings = [...discovery.warnings, ...initialState.preLoadWarnings];
const renderState =
preLoadWarnings.length > 0
? {
...initialState,
preLoadWarnings,
}
: initialState;
for (const warning of preLoadWarnings) {
io.stderr.write(`${warning}\n`);
}
const project = await loadProject({ projectDir: args.projectDir });
await applyNotionPickerWriteback(project, args.connectionId, args.rootPageIds);
io.stdout.write(`Connection: ${args.connectionId}\n`);
io.stdout.write(`rootPageIds: ${args.rootPageIds.length}\n`);
io.stdout.write('crawlMode: selected_roots\n');
return 0;
const workspaceLabel = await resolveNotionWorkspaceLabel(api, args.connectionId);
const result = await (deps.renderPicker ?? renderNotionPickerTui)(
{
initialState: renderState,
connectionId: args.connectionId,
workspaceLabel,
cappedAtCount: discovery.cappedAtCount,
currentCrawlMode: crawlMode,
},
io as NotionPickerTuiIo,
);
if (result.kind === 'quit') {
return { kind: 'back' };
}
if (result.rootPageIds.length === 0) {
return { kind: 'unavailable', message: 'Notion picker did not return any selected pages.' };
}
return { kind: 'selected', rootPageIds: result.rootPageIds };
} catch (error) {
io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
return 1;
return { kind: 'unavailable', message: error instanceof Error ? error.message : String(error) };
}
}

8
packages/cli/src/print-command-tree.test.ts
View file

@ -16,7 +16,13 @@ describe('renderKtxCommandTree', () => {
expect(topLevel).toContain(expected);
}
expect(output).toContain('│ ├── test <connectionId>');
expect(output).toContain('│ └── test <connectionId>');
expect(output).not.toContain('│ ├── add');
expect(output).not.toContain('│ ├── remove');
expect(output).not.toContain('│ ├── map');
expect(output).not.toContain('│ ├── mapping');
expect(output).not.toContain('│ ├── metabase');
expect(output).not.toContain('│ ├── notion');
});
it('ends with a single trailing newline', () => {

28
packages/cli/src/project-dir.test.ts
View file

@ -33,11 +33,9 @@ describe('project directory defaults', () => {
const connection = vi.fn(async () => 0);
const doctor = vi.fn(async () => 0);
const ingest = vi.fn(async () => 0);
const publicIngest = vi.fn(async () => 0);
const scan = vi.fn(async () => 0);
const setup = vi.fn(async () => 0);
const agent = vi.fn(async () => 0);
const deps: KtxCliDeps = { agent, connection, doctor, ingest, publicIngest, scan, setup };
const deps: KtxCliDeps = { connection, doctor, ingest, scan, setup };
const cases: Array<{
argv: string[];
@ -59,28 +57,22 @@ describe('project directory defaults', () => {
},
{
argv: ['ingest', 'status', 'run-1'],
spy: publicIngest,
expected: { command: 'status', projectDir: '/tmp/ktx-env-project', runId: 'run-1' },
spy: ingest,
expected: { command: 'status', projectDir: '/tmp/ktx-env-project', runId: 'run-1', outputMode: 'plain' },
expectedStderr: 'Project: /tmp/ktx-env-project\n',
},
{
argv: ['setup', '--no-input'],
spy: setup,
expected: { command: 'run', projectDir: '/tmp/ktx-env-project' },
expectedStderr: 'Project: /tmp/ktx-env-project\n',
expectedStderr: '',
},
{
argv: ['dev', 'scan', 'warehouse'],
argv: ['scan', 'warehouse'],
spy: scan,
expected: { command: 'run', projectDir: '/tmp/ktx-env-project', connectionId: 'warehouse' },
expectedStderr: 'Project: /tmp/ktx-env-project\n',
},
{
argv: ['agent', 'tools', '--json'],
spy: agent,
expected: { command: 'tools', projectDir: '/tmp/ktx-env-project' },
expectedStderr: '',
},
];
for (const item of cases) {
@ -95,16 +87,16 @@ describe('project directory defaults', () => {
process.env.KTX_PROJECT_DIR = '/tmp/ktx-env-project';
const scan = vi.fn(async () => 0);
const publicIngest = vi.fn(async () => 0);
const ingest = vi.fn(async () => 0);
const scanIo = makeIo();
const ingestIo = makeIo();
await expect(
runKtxCli(['--project-dir', '/tmp/ktx-explicit-project', 'dev', 'scan', 'warehouse'], scanIo.io, { scan }),
runKtxCli(['--project-dir', '/tmp/ktx-explicit-project', 'scan', 'warehouse'], scanIo.io, { scan }),
).resolves.toBe(0);
await expect(
runKtxCli(['ingest', 'status', 'run-1', '--project-dir=/tmp/ktx-explicit-project'], ingestIo.io, {
publicIngest,
ingest,
}),
).resolves.toBe(0);
@ -112,7 +104,7 @@ describe('project directory defaults', () => {
expect.objectContaining({ command: 'run', projectDir: '/tmp/ktx-explicit-project' }),
scanIo.io,
);
expect(publicIngest).toHaveBeenCalledWith(
expect(ingest).toHaveBeenCalledWith(
expect.objectContaining({ command: 'status', projectDir: '/tmp/ktx-explicit-project' }),
ingestIo.io,
);
@ -139,7 +131,7 @@ describe('project directory defaults', () => {
try {
process.chdir(nestedDir);
await expect(runKtxCli(['dev', 'scan', 'warehouse'], testIo.io, { scan })).resolves.toBe(0);
await expect(runKtxCli(['scan', 'warehouse'], testIo.io, { scan })).resolves.toBe(0);
} finally {
process.chdir(originalCwd);
await rm(root, { recursive: true, force: true });

5
packages/cli/src/project-dir.ts
View file

@ -1,5 +0,0 @@
import { resolve } from 'node:path';
export function resolveProjectDir(projectDir?: string, fallback = '.'): string {
return resolve(projectDir ?? fallback);
}

6
packages/cli/src/public-ingest.test.ts
View file

@ -57,7 +57,7 @@ describe('buildPublicIngestPlan', () => {
driver: 'notion',
operation: 'source-ingest',
adapter: 'notion',
debugCommand: 'ktx dev ingest run --connection-id docs --adapter notion --debug',
debugCommand: 'ktx ingest run --connection-id docs --adapter notion --debug',
steps: ['source-ingest', 'memory-update'],
},
{
@ -65,7 +65,7 @@ describe('buildPublicIngestPlan', () => {
driver: 'metabase',
operation: 'source-ingest',
adapter: 'metabase',
debugCommand: 'ktx dev ingest run --connection-id prod_metabase --adapter metabase --debug',
debugCommand: 'ktx ingest run --connection-id prod_metabase --adapter metabase --debug',
steps: ['source-ingest', 'memory-update'],
},
],
@ -76,7 +76,7 @@ describe('buildPublicIngestPlan', () => {
const project = projectWithConnections({ warehouse: { driver: 'postgres' } });
expect(() => buildPublicIngestPlan(project, { projectDir: '/tmp/project', all: false })).toThrow(
'ktx ingest requires <connectionId> or --all in this release',
'Context build requires a connection id or all targets',
);
});

Some files were not shown because too many files have changed in this diff Show more