apunkt/ktx
1
0
Fork 0
mirror of https://github.com/Kaelio/ktx.git synced 2026-06-10 08:05:14 +02:00
Code Issues Projects Releases Packages Wiki Activity

Merge remote-tracking branch 'origin/main' into audit-ktx-yaml-params

Browse source 
# Conflicts:
#	packages/cli/src/doctor.ts
This commit is contained in:
Andrey Avtomonov 2026-05-14 01:01:03 +02:00
commit 1f648345f1
51 changed files with 4443 additions and 2262 deletions
Download patch file Download diff file Expand all files Collapse all files

50
.github/workflows/ci.yml vendored
View file

@ -15,6 +15,46 @@ concurrency:
cancel-in-progress: true
jobs:
pre-commit-checks:
name: Pre-commit checks
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Setup pnpm
uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8
with:
run_install: false
- name: Setup Node.js
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
with:
node-version: "24"
cache: "pnpm"
cache-dependency-path: "pnpm-lock.yaml"
- name: Install TypeScript dependencies
run: pnpm install --frozen-lockfile
- name: Setup Python
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
with:
python-version: "3.13"
- name: Setup uv
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
with:
version: "0.11.11"
enable-cache: true
cache-dependency-glob: "uv.lock"
- name: Install Python dependencies
run: uv sync --all-packages --all-groups
- name: Run pre-commit hooks
run: uv run pre-commit run --all-files
typescript-checks:
name: TypeScript checks
runs-on: ubuntu-latest
@ -23,7 +63,7 @@ jobs:
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Setup pnpm
uses: pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0 # v6.0.7
uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8
with:
run_install: false
@ -51,7 +91,7 @@ jobs:
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Setup pnpm
uses: pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0 # v6.0.7
uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8
with:
run_install: false
@ -79,7 +119,7 @@ jobs:
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Setup pnpm
uses: pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0 # v6.0.7
uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8
with:
run_install: false
@ -107,7 +147,7 @@ jobs:
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Setup pnpm
uses: pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0 # v6.0.7
uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8
with:
run_install: false
@ -156,7 +196,7 @@ jobs:
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Setup pnpm
uses: pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0 # v6.0.7
uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8
with:
run_install: false

2
.github/workflows/release.yml vendored
View file

@ -24,7 +24,7 @@ jobs:
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Setup pnpm
uses: pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0 # v6.0.7
uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8
with:
run_install: false

1
LICENSE
View file

@ -199,4 +199,3 @@
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

94
docs-site/content/docs/cli-reference/ktx-setup.mdx
View file

@ -18,8 +18,6 @@ ktx setup [options]
| Flag | Description | Default |
|------|-------------|---------|
| `--project-dir <path>` | KTX project directory | `KTX_PROJECT_DIR`, nearest `ktx.yaml`, or cwd |
| `--new` | Create a new KTX project before setup | `false` |
| `--existing` | Use an existing KTX project | `false` |
| `--yes` | Accept safe defaults in non-interactive setup | `false` |
| `--no-input` | Disable interactive terminal input | — |
@ -29,76 +27,11 @@ ktx setup [options]
|------|-------------|---------|
| `--agents` | Install agent integration only | `false` |
| `--target <target>` | Agent target (`claude-code`, `codex`, `cursor`, `opencode`, `universal`) | — |
| `--agent-scope <scope>` | Agent install scope (`project` or `global`) | `project` |
| `--project` | Install agent integration into the project scope | `false` |
| `--global` | Install agent integration into the global target scope (Claude Code and Codex only) | `false` |
| `--skip-agents` | Leave agent integration incomplete for now | `false` |
### LLM Configuration
| Flag | Description | Default |
|------|-------------|---------|
| `--anthropic-api-key-env <name>` | Environment variable containing the Anthropic API key | — |
| `--anthropic-api-key-file <path>` | File containing the Anthropic API key | — |
| `--anthropic-model <model>` | Anthropic model ID to validate and save | — |
| `--skip-llm` | Leave LLM setup incomplete for now | `false` |
### Embedding Configuration
| Flag | Description | Default |
|------|-------------|---------|
| `--embedding-backend <backend>` | Embedding backend (`openai` or `sentence-transformers`) | — |
| `--embedding-api-key-env <name>` | Environment variable containing the embedding provider API key | — |
| `--embedding-api-key-file <path>` | File containing the embedding provider API key | — |
| `--skip-embeddings` | Leave embedding setup incomplete for now | `false` |
### Database Configuration
| Flag | Description | Default |
|------|-------------|---------|
| `--database <driver>` | Database driver to configure; repeatable (`sqlite`, `postgres`, `mysql`, `clickhouse`, `sqlserver`, `bigquery`, `snowflake`) | — |
| `--database-connection-id <id>` | Existing or new connection id; repeatable | — |
| `--new-database-connection-id <id>` | Connection id for one new database connection | — |
| `--database-url <url>` | URL, `env:NAME`, or `file:/path` for one new URL-style database connection | — |
| `--database-schema <schema>` | Database schema to include; repeatable | — |
| `--skip-databases` | Leave database setup incomplete | `false` |
### Historic SQL
| Flag | Description | Default |
|------|-------------|---------|
| `--enable-historic-sql` | Enable Historic SQL when the selected database supports it | `false` |
| `--disable-historic-sql` | Disable Historic SQL for the selected database | `false` |
| `--historic-sql-window-days <number>` | Historic SQL query-history window in days | — |
| `--historic-sql-min-executions <number>` | Minimum executions for a Historic SQL template | — |
| `--historic-sql-min-calls <number>` | Alias for `--historic-sql-min-executions` for one release | — |
| `--historic-sql-service-account-pattern <pattern>` | Historic SQL service-account regex; repeatable | — |
| `--historic-sql-redaction-pattern <pattern>` | Historic SQL SQL-literal redaction regex; repeatable | — |
### Context Source Configuration
| Flag | Description | Default |
|------|-------------|---------|
| `--source <type>` | Source connector type (`dbt`, `metricflow`, `metabase`, `looker`, `lookml`, `notion`) | — |
| `--source-connection-id <id>` | Connection id for source setup | — |
| `--source-path <path>` | Local source path for dbt, MetricFlow, or LookML | — |
| `--source-git-url <url>` | Git URL for dbt, MetricFlow, or LookML | — |
| `--source-branch <branch>` | Git branch for source setup | — |
| `--source-subpath <path>` | Repo subpath for source setup | — |
| `--source-auth-token-ref <ref>` | `env:` or `file:` credential ref for source repo auth | — |
| `--source-url <url>` | Source service URL for Metabase or Looker | — |
| `--source-api-key-ref <ref>` | `env:` or `file:` API key ref for Metabase or Notion | — |
| `--source-client-id <id>` | Looker client id | — |
| `--source-client-secret-ref <ref>` | `env:` or `file:` Looker client secret ref | — |
| `--source-warehouse-connection-id <id>` | Mapped warehouse connection id | — |
| `--source-project-name <name>` | dbt project name override | — |
| `--source-profiles-path <path>` | dbt profiles path | — |
| `--source-target <target>` | dbt target or source-specific mapping target | — |
| `--metabase-database-id <id>` | Metabase database id to map | — |
| `--notion-crawl-mode <mode>` | Notion crawl mode (`all_accessible` or `selected_roots`) | — |
| `--notion-root-page-id <id>` | Notion root page id; repeatable | — |
| `--skip-initial-source-ingest` | Validate source setup without building source context during setup | `false` |
| `--skip-sources` | Mark optional source setup complete with no sources | `false` |
The setup wizard is the public configuration interface. It prompts for LLM
credentials, embeddings, database connections, context sources, Historic SQL,
and agent integration when those values are needed.
## Examples
@ -106,17 +39,8 @@ ktx setup [options]
# Run the interactive setup wizard
ktx setup
# Create a new project and run setup
ktx setup --new
# Resume setup in an existing project
ktx setup --existing
# Non-interactive setup with Anthropic key from environment
ktx setup --yes --anthropic-api-key-env ANTHROPIC_API_KEY
# Set up a Postgres connection
ktx setup --database postgres --database-url "env:DATABASE_URL"
# Run setup for a specific project directory
ktx setup --project-dir ./analytics
# Install agent integration for Claude Code only
ktx setup --agents --target claude-code
@ -124,12 +48,6 @@ ktx setup --agents --target claude-code
# Install agent integration globally for Codex
ktx setup --agents --target codex --global
# Add a dbt source from a local path
ktx setup --source dbt --source-path ./my-dbt-project
# Skip optional steps for a minimal setup
ktx setup --skip-sources --skip-agents
# Check setup readiness
ktx status
```
@ -156,5 +74,5 @@ Agent integration ready: yes (codex:project)
|-------|-------|----------|
| Setup resumes an unexpected project | `KTX_PROJECT_DIR` or nearest `ktx.yaml` points to another directory | Pass `--project-dir <path>` explicitly |
| Health check for model fails | Provider key or model id is invalid | Set the correct environment variable or secret file and rerun setup |
| Setup cannot run in CI | Interactive prompts need a TTY | Use `--yes --no-input` with explicit flags for required values |
| Setup cannot run in CI | Interactive prompts need a TTY | Run setup interactively before CI, or provide a fixture `ktx.yaml` for automated tests |
| Agent integration missing | Setup skipped the agents step | Run `ktx setup --agents --target <target>` |

2
docs-site/content/docs/getting-started/quickstart.mdx
View file

@ -242,7 +242,7 @@ Agent integration ready: yes (claude-code:project)
| Local embeddings hang or fail | The managed Python runtime cannot start or the local model runtime is unavailable | Install `uv`, run `ktx dev runtime status`, then run `ktx dev runtime install --feature local-embeddings --yes` and rerun setup |
| Database connection test fails | Credentials, network access, warehouse, database, or schema value is wrong | Test the same URL with the database's native client, then rerun `ktx setup` and reconfigure the connection |
| `KTX context built: no` in `ktx status` | Setup saved configuration but did not build context | Run `ktx setup` and choose to build context now |
| Agent integration is incomplete | Setup skipped the agents step or the target was not installed | Run `ktx setup --agents --target codex --project` using the target you need |
| Agent integration is incomplete | Setup skipped the agents step or the target was not installed | Run `ktx setup --agents --target codex` using the target you need |
## Next steps

6
docs-site/content/docs/integrations/primary-sources.mdx
View file

@ -37,7 +37,7 @@ The most full-featured connector. Supports schema introspection, foreign key det
connections:
my-postgres:
driver: postgres
url: postgresql://user:password@host:5432/database
url: env:DATABASE_URL
schema: public
```
@ -326,7 +326,7 @@ Standard MySQL/MariaDB connector with full foreign key support and schema intros
connections:
my-mysql:
driver: mysql
url: mysql://user:password@host:3306/database
url: env:MYSQL_DATABASE_URL
```
Or with individual fields:
@ -383,7 +383,7 @@ Connects to Microsoft SQL Server and Azure SQL. Supports multi-schema scanning w
connections:
my-sqlserver:
driver: sqlserver
url: mssql://user:password@host:1433/database?trustServerCertificate=true
url: env:SQLSERVER_DATABASE_URL
```
Or with individual fields:

32
docs/superpowers/plans/2026-05-11-historic-sql-cross-dialect-readiness.md
View file

@ -40,37 +40,37 @@ This plan does not update `examples/postgres-historic/README.md` or `examples/po
Modify:
- `packages/context/src/ingest/adapters/historic-sql/types.ts`
- `packages/context/src/ingest/adapters/historic-sql/types.ts`
Adds optional probe `info` notes and lets injected historic-SQL dependencies use any reader/query client pair while preserving the existing Postgres-specific option.
- `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.ts`
- `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.ts`
Moves low `pg_stat_statements.max` from `warnings` to `info`.
- `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.test.ts`
- `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.test.ts`
Locks `track = none` as warning and low `max` as info.
- `packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.ts`
- `packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.ts`
Returns `{ warnings: [], info: [] }` from `probe()`.
- `packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.test.ts`
- `packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.test.ts`
Locks the BigQuery probe return object.
- `packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.ts`
- `packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.ts`
Returns `{ warnings: [], info: [] }` from `probe()`.
- `packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.test.ts`
- `packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.test.ts`
Locks the Snowflake probe return object.
- `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts`
- `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts`
Updates test readers to return the normalized probe shape.
- `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts`
- `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts`
Updates test readers to return the normalized probe shape.
- `packages/context/src/ingest/local-adapters.ts`
- `packages/context/src/ingest/local-adapters.ts`
Accepts generic historic-SQL reader/query-client dependencies while keeping `postgresQueryClient` as the compatibility input used by current callers.
- `packages/context/src/ingest/local-adapters.test.ts`
- `packages/context/src/ingest/local-adapters.test.ts`
Verifies generic reader/query-client injection and the existing Postgres compatibility path.
- `packages/cli/src/local-adapters.ts`
- `packages/cli/src/local-adapters.ts`
Chooses Postgres, BigQuery, or Snowflake historic-SQL readers/query clients from the configured connection.
- `packages/cli/src/local-adapters.test.ts`
- `packages/cli/src/local-adapters.test.ts`
Adds direct tests for CLI local adapter registration for Postgres, BigQuery, and Snowflake.
- `packages/cli/src/historic-sql-doctor.ts`
- `packages/cli/src/historic-sql-doctor.ts`
Treats info-only Postgres probe notes as a passing doctor check, and warnings as warnings.
- `packages/cli/src/historic-sql-doctor.test.ts`
- `packages/cli/src/historic-sql-doctor.test.ts`
Verifies low `pg_stat_statements.max` is pass/detail, while `track = none` remains warn.
- `packages/cli/src/doctor.test.ts`
- `packages/cli/src/doctor.test.ts`
Updates the project doctor integration expectation for the new info-only behavior.
## Task 1: Normalize Historic-SQL Probe Results

2
docs/superpowers/plans/2026-05-11-historic-sql-end-to-end-retrieval-acceptance.md
View file

@ -44,7 +44,7 @@ Remaining acceptance gap this plan covers:
Create:
- `packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts`
- `packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts`
Owns the end-to-end local regression for the redesigned historic-SQL pipeline. It uses the real adapter and local ingest runner, with fake deterministic reader/analysis/agent components so the test does not need a live database or LLM provider.
## Task 1: Add Real-Adapter Local Ingest Acceptance Coverage

42
docs/superpowers/plans/2026-05-11-historic-sql-foundations.md
View file

@ -41,50 +41,50 @@ The next plan after this one should cover search enrichment from spec §6.2.3-§
Create:
- `packages/context/src/ingest/adapters/historic-sql/skill-schemas.ts`
- `packages/context/src/ingest/adapters/historic-sql/skill-schemas.ts`
Owns the shared zod schemas for historic-SQL LLM outputs.
- `packages/context/src/ingest/adapters/historic-sql/skill-schemas.test.ts`
- `packages/context/src/ingest/adapters/historic-sql/skill-schemas.test.ts`
Locks schema acceptance, JSON schema generation, and future-key tolerance.
- `python/ktx-daemon/src/ktx_daemon/sql_analysis.py`
- `python/ktx-daemon/src/ktx_daemon/sql_analysis.py`
Implements batch sqlglot parsing for table and clause-level column extraction.
- `python/ktx-daemon/tests/test_sql_analysis.py`
- `python/ktx-daemon/tests/test_sql_analysis.py`
Tests batch parser behavior without FastAPI.
Modify:
- `packages/context/src/ingest/index.ts`
- `packages/context/src/ingest/index.ts`
Exports the new historic-SQL skill schemas.
- `packages/context/src/sl/types.ts`
- `packages/context/src/sl/types.ts`
Adds `usage?: TableUsageOutput` to `SemanticLayerSource`.
- `packages/context/src/sl/schemas.ts`
- `packages/context/src/sl/schemas.ts`
Accepts `usage` in standalone and overlay semantic-layer source validation.
- `packages/context/src/sl/semantic-layer.service.ts`
- `packages/context/src/sl/semantic-layer.service.ts`
Projects manifest `usage` onto `SemanticLayerSource` and composes overlay usage intentionally.
- `packages/context/src/sl/semantic-layer.service.test.ts`
- `packages/context/src/sl/semantic-layer.service.test.ts`
Tests source schema acceptance, manifest projection, and overlay composition.
- `packages/context/src/ingest/adapters/live-database/manifest.ts`
- `packages/context/src/ingest/adapters/live-database/manifest.ts`
Adds `LiveDatabaseManifestTableEntry.usage`, existing-usage inputs, and `mergeUsagePreservingExternal()`.
- `packages/context/src/ingest/adapters/live-database/manifest.test.ts`
- `packages/context/src/ingest/adapters/live-database/manifest.test.ts`
Tests scan-managed usage replacement while preserving external keys.
- `packages/context/src/scan/local-enrichment-artifacts.ts`
- `packages/context/src/scan/local-enrichment-artifacts.ts`
Loads existing manifest usage and passes it through scan manifest rebuilds.
- `packages/context/src/scan/local-enrichment-artifacts.test.ts`
- `packages/context/src/scan/local-enrichment-artifacts.test.ts`
Tests that structural scan rewrites preserve existing usage.
- `python/ktx-daemon/src/ktx_daemon/app.py`
- `python/ktx-daemon/src/ktx_daemon/app.py`
Registers `/sql/analyze-batch`.
- `python/ktx-daemon/tests/test_app.py`
- `python/ktx-daemon/tests/test_app.py`
Tests the FastAPI endpoint.
- `packages/context/src/sql-analysis/ports.ts`
- `packages/context/src/sql-analysis/ports.ts`
Adds batch analysis types and `SqlAnalysisPort.analyzeBatch()`.
- `packages/context/src/sql-analysis/index.ts`
- `packages/context/src/sql-analysis/index.ts`
Exports the new batch analysis types.
- `packages/context/src/sql-analysis/http-sql-analysis-port.ts`
- `packages/context/src/sql-analysis/http-sql-analysis-port.ts`
Maps `/sql/analyze-batch` request and response payloads.
- `packages/context/src/sql-analysis/http-sql-analysis-port.test.ts`
- `packages/context/src/sql-analysis/http-sql-analysis-port.test.ts`
Tests HTTP mapping and malformed response rejection.
- `packages/cli/src/managed-python-http.test.ts`
- `packages/cli/src/managed-python-http.test.ts`
Verifies the managed daemon wrapper routes `analyzeBatch()`.
- Existing test files with `SqlAnalysisPort` object literals
- Existing test files with `SqlAnalysisPort` object literals
Add a no-op `analyzeBatch: async () => new Map()` while legacy paths still use `analyzeForFingerprint()`.
## Task 1: Add Historic SQL Skill Schemas

8
docs/superpowers/plans/2026-05-11-historic-sql-pattern-shard-smoke-docs.md
View file

@ -39,13 +39,13 @@ Remaining gap this plan fixes:
## File Structure
- Modify `scripts/examples-docs.test.mjs`
- Modify `scripts/examples-docs.test.mjs`
Pins docs and smoke script to the sharded pattern WorkUnit contract.
- Modify `examples/postgres-historic/scripts/smoke.sh`
- Modify `examples/postgres-historic/scripts/smoke.sh`
Validates `patterns-input/part-*.json` shard files and `historic-sql-patterns-part-*` stage-only WorkUnits.
- Modify `examples/postgres-historic/README.md`
- Modify `examples/postgres-historic/README.md`
Documents `patterns-input.json` as the full audit artifact and `patterns-input/part-*.json` as bounded pattern WorkUnit input.
- Modify `examples/README.md`
- Modify `examples/README.md`
Updates the short example catalog entry with the same audit-vs-shard wording.
### Task 1: Pin Example Tests To Pattern Shards

18
docs/superpowers/plans/2026-05-11-historic-sql-pattern-workunit-sharding.md
View file

@ -30,23 +30,23 @@ No existing spec-derived plan is currently unimplemented in this worktree. This
## File Structure
- Create `packages/context/src/ingest/adapters/historic-sql/pattern-inputs.ts`
- Create `packages/context/src/ingest/adapters/historic-sql/pattern-inputs.ts`
Owns deterministic pattern audit ordering, cross-table candidate filtering, byte-bounded shard creation, shard path constants, and shard path detection.
- Create `packages/context/src/ingest/adapters/historic-sql/pattern-inputs.test.ts`
- Create `packages/context/src/ingest/adapters/historic-sql/pattern-inputs.test.ts`
Covers deterministic shard ordering, single-table exclusion from WorkUnit shards, byte limits, and oversize-template manifest warnings.
- Modify `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts`
- Modify `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts`
Writes full `patterns-input.json` plus bounded `patterns-input/part-0001.json` shard files, and appends shard warnings to `manifest.json`.
- Modify `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts`
- Modify `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts`
Adds a regression for audit file preservation and sharded WorkUnit input creation.
- Modify `packages/context/src/ingest/adapters/historic-sql/chunk-unified.ts`
- Modify `packages/context/src/ingest/adapters/historic-sql/chunk-unified.ts`
Emits one patterns WorkUnit per changed shard path, treats root `patterns-input.json` as audit-only, and includes shard paths in the scope descriptor and eviction calculation.
- Modify `packages/context/src/ingest/adapters/historic-sql/chunk-unified.test.ts`
- Modify `packages/context/src/ingest/adapters/historic-sql/chunk-unified.test.ts`
Updates root-file expectations and adds multi-shard diff behavior.
- Modify `packages/context/skills/historic_sql_patterns/SKILL.md`
- Modify `packages/context/skills/historic_sql_patterns/SKILL.md`
Tells the skill to read the exact pattern shard in `rawFiles` and emit evidence with that shard as `rawPath`.
- Modify `packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts`
- Modify `packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts`
Updates the fake agent to emit pattern evidence for `historic-sql-patterns-part-0001`.
- Modify `packages/context/src/ingest/ingest-runtime-assets.test.ts`
- Modify `packages/context/src/ingest/ingest-runtime-assets.test.ts`
Keeps packaged skill assertions aligned with sharded pattern file guidance.
## Task 1: Add Pattern Input Sharding Helper

12
docs/superpowers/plans/2026-05-11-historic-sql-redaction-hardening.md
View file

@ -55,16 +55,16 @@ Remaining spec gap this plan covers:
Create:
- `packages/context/src/ingest/adapters/historic-sql/redaction.ts`
- `packages/context/src/ingest/adapters/historic-sql/redaction.ts`
Owns compilation and application of historic-SQL SQL-text redaction patterns. Supports JavaScript regex strings and the documented `(?i)` case-insensitive prefix used by setup tests/docs.
- `packages/context/src/ingest/adapters/historic-sql/redaction.test.ts`
- `packages/context/src/ingest/adapters/historic-sql/redaction.test.ts`
Tests raw regex replacement, `(?i)` compatibility, empty config behavior, and invalid-pattern diagnostics.
Modify:
- `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts`
- `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts`
Compiles `config.redactionPatterns` once per fetch. Keeps original SQL for filtering and `SqlAnalysisPort.analyzeBatch()`, then stores redacted SQL in `ParsedTemplate.template.canonicalSql` before `toStagedTable()` and `toPatternsInput()` serialize files.
- `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts`
- `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts`
Adds a regression proving raw secrets are absent from staged artifacts while `analyzeBatch()` still receives the original SQL.
## Task 1: Add Historic SQL Redaction Helper
@ -89,7 +89,7 @@ describe('historic-SQL redaction', () => {
]);
const sql =
"select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'";
"select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; // pragma: allowlist secret
expect(redactHistoricSqlText(sql, redactors)).toBe(
"select * from public.api_events where api_key = '[REDACTED]' and note = '[REDACTED]'",
@ -202,7 +202,7 @@ Append this test inside the existing `describe('stageHistoricSqlAggregatedSnapsh
it('redacts configured SQL substrings in staged artifacts while analyzing original SQL', async () => {
const stagedDir = await tempDir();
const originalSql =
"select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'";
"select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; // pragma: allowlist secret
const reader: HistoricSqlReader = {
async probe() {
return { warnings: [], info: [] };

22
docs/superpowers/plans/2026-05-11-historic-sql-search-enrichment.md
View file

@ -37,27 +37,27 @@ This plan does not rewrite the historic-SQL adapter, readers, skills, projection
Modify:
- `packages/context/src/sl/sl-search.service.ts`
- `packages/context/src/sl/sl-search.service.ts`
Adds usage narrative, frequency, filters, group-bys, joins, and stale marker to the canonical SL search text. Preserves snippets returned by repository search for direct `SlSearchService.search()` callers.
- `packages/context/src/sl/sl-search.service.test.ts`
- `packages/context/src/sl/sl-search.service.test.ts`
Tests usage search-text content and direct service snippet pass-through.
- `packages/context/src/sl/ports.ts`
- `packages/context/src/sl/ports.ts`
Extends `SlSourcesIndexPort.search()` rows with optional `snippet`.
- `packages/context/src/sl/sqlite-sl-sources-index.ts`
- `packages/context/src/sl/sqlite-sl-sources-index.ts`
Adds FTS5 `snippet()` selection to lexical candidate search and direct index search.
- `packages/context/src/sl/sqlite-sl-sources-index.test.ts`
- `packages/context/src/sl/sqlite-sl-sources-index.test.ts`
Locks snippet behavior for both direct search and lexical lane candidates.
- `packages/context/src/sl/local-sl.ts`
- `packages/context/src/sl/local-sl.ts`
Adds `frequencyTier` and `snippet` to query-mode `LocalSlSourceSearchResult`; collects snippets from the lexical lane and hydrates frequency from `SemanticLayerSource.usage`.
- `packages/context/src/sl/local-sl.test.ts`
- `packages/context/src/sl/local-sl.test.ts`
Tests that usage-only terms can find a source and that results include `frequencyTier` and FTS snippet.
- `packages/context/src/sl/pglite-sl-search-prototype.ts`
- `packages/context/src/sl/pglite-sl-search-prototype.ts`
Propagates `frequencyTier` for the prototype backend so the shared result type stays truthful.
- `packages/context/src/mcp/types.ts`
- `packages/context/src/mcp/types.ts`
Adds `frequencyTier` and `snippet` to `KtxSemanticLayerSourceSummary`.
- `packages/context/src/mcp/local-project-ports.ts`
- `packages/context/src/mcp/local-project-ports.ts`
Includes `frequencyTier` and `snippet` in `semanticLayer.listSources()` output.
- `packages/context/src/mcp/local-project-ports.test.ts`
- `packages/context/src/mcp/local-project-ports.test.ts`
Tests the agent/MCP-facing list response.
## Task 1: Index Historic SQL Usage In SL Search Text

50
docs/superpowers/plans/2026-05-11-historic-sql-skills-projection-cutover.md
View file

@ -52,58 +52,58 @@ Still not implemented:
Create:
- `packages/context/src/ingest/adapters/historic-sql/evidence.ts`
- `packages/context/src/ingest/adapters/historic-sql/evidence.ts`
Owns typed evidence envelopes, ignored evidence path helpers, and load/write helpers for table usage and pattern evidence.
- `packages/context/src/ingest/adapters/historic-sql/evidence.test.ts`
- `packages/context/src/ingest/adapters/historic-sql/evidence.test.ts`
Tests evidence schema validation, path normalization, and loader rejection of malformed evidence.
- `packages/context/src/ingest/adapters/historic-sql/evidence-tool.ts`
- `packages/context/src/ingest/adapters/historic-sql/evidence-tool.ts`
Adds `emit_historic_sql_evidence`, the only write tool the two new historic-SQL skills use.
- `packages/context/src/ingest/adapters/historic-sql/evidence-tool.test.ts`
- `packages/context/src/ingest/adapters/historic-sql/evidence-tool.test.ts`
Tests the tool writes ignored run-local JSON with `skipLock: true` and rejects non-historic ingest sessions.
- `packages/context/src/ingest/adapters/historic-sql/projection.ts`
- `packages/context/src/ingest/adapters/historic-sql/projection.ts`
Projects table usage evidence into manifest shards, writes pattern wiki pages, marks stale usage/pages, and deletes legacy query pages.
- `packages/context/src/ingest/adapters/historic-sql/projection.test.ts`
- `packages/context/src/ingest/adapters/historic-sql/projection.test.ts`
Tests `_schema` merge, stale usage, pattern slug reuse, stale page tagging, archive movement, and legacy page cleanup.
- `packages/context/src/ingest/adapters/historic-sql/post-processor.ts`
- `packages/context/src/ingest/adapters/historic-sql/post-processor.ts`
Implements `IngestBundlePostProcessorPort` for the deterministic projection phase.
- `packages/context/src/ingest/adapters/historic-sql/post-processor.test.ts`
- `packages/context/src/ingest/adapters/historic-sql/post-processor.test.ts`
Tests post-processor path resolution from `workdir`, `connectionId`, `sourceKey`, and `syncId`.
- `packages/context/skills/historic_sql_table_digest/SKILL.md`
- `packages/context/skills/historic_sql_table_digest/SKILL.md`
Skill for one changed `tables/*.json` WorkUnit; emits one table usage evidence object.
- `packages/context/skills/historic_sql_patterns/SKILL.md`
- `packages/context/skills/historic_sql_patterns/SKILL.md`
Skill for `patterns-input.json`; emits one pattern evidence object per recurring cross-table intent.
Modify:
- `packages/context/src/ingest/adapters/historic-sql/types.ts`
- `packages/context/src/ingest/adapters/historic-sql/types.ts`
Keep only unified config/staged schemas and reader contracts; extend config preprocessing for existing `serviceAccountUserPatterns` and `minCalls` aliases.
- `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts`
- `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts`
Add `staleArchiveAfterDays` to `manifest.json` so projection can archive stale pattern pages deterministically.
- `packages/context/src/ingest/adapters/historic-sql/chunk-unified.ts`
- `packages/context/src/ingest/adapters/historic-sql/chunk-unified.ts`
Keep the same WorkUnits, but mention `emit_historic_sql_evidence` in `notes`.
- `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.ts`
- `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.ts`
Switch production fetch/chunk/scope to the unified hot path, replace skills, remove legacy triage support, and run legacy PGSS baseline cache cleanup.
- `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts`
- `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts`
Rewrite around unified staging and new skills.
- `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.ts`
- `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.ts`
Inline the PGSS probe logic so `postgres-pgss-query-history-reader.ts` can be deleted.
- `packages/context/src/ingest/local-adapters.ts`
- `packages/context/src/ingest/local-adapters.ts`
Use `PostgresPgssReader` for local Postgres historic SQL and return unified pull config.
- `packages/context/src/ingest/local-bundle-runtime.ts`
- `packages/context/src/ingest/local-bundle-runtime.ts`
Add the source-specific evidence tool to historic-SQL WorkUnits and register the historic-SQL post-processor.
- `packages/context/src/ingest/ingest-runtime-assets.test.ts`
- `packages/context/src/ingest/ingest-runtime-assets.test.ts`
Replace old skill asset assertions with the two new skills.
- `packages/context/src/memory/memory-runtime-assets.test.ts`
- `packages/context/src/memory/memory-runtime-assets.test.ts`
Replace old historic-SQL skill heading with the two new skill headings.
- `packages/context/src/package-exports.test.ts`
- `packages/context/src/package-exports.test.ts`
Remove legacy export assertions and add evidence/projection export assertions.
- `packages/context/src/ingest/index.ts`
- `packages/context/src/ingest/index.ts`
Export new evidence/projection/post-processor helpers and remove legacy historic-SQL exports.
- `packages/cli/src/setup-databases.ts` and `packages/cli/src/historic-sql-doctor.ts`
- `packages/cli/src/setup-databases.ts` and `packages/cli/src/historic-sql-doctor.ts`
Import `PostgresPgssReader` instead of `PostgresPgssQueryHistoryReader`.
- `packages/cli/src/commands/setup-commands.ts`, `packages/cli/src/index.test.ts`, `packages/cli/src/setup-databases.test.ts`
- `packages/cli/src/commands/setup-commands.ts`, `packages/cli/src/index.test.ts`, `packages/cli/src/setup-databases.test.ts`
Rename generated config to `minExecutions` while accepting the old `--historic-sql-min-calls` flag for one release.
- `packages/context/prompts/skills/page_triage_classifier.md`, `packages/context/src/ingest/page-triage/page-triage.service.test.ts`, `packages/context/src/ingest/ingest-prompts.test.ts`
- `packages/context/prompts/skills/page_triage_classifier.md`, `packages/context/src/ingest/page-triage/page-triage.service.test.ts`, `packages/context/src/ingest/ingest-prompts.test.ts`
Remove historic-SQL template triage examples because the new adapter no longer uses page triage.
Delete:

213
packages/cli/src/commands/setup-commands.ts
View file

@ -64,13 +64,6 @@ function sourceType(value: string): KtxSetupSourceType {
throw new InvalidArgumentError(`invalid choice '${value}'`);
}
function agentScope(value: string): 'project' | 'global' {
if (value === 'project' || value === 'global') {
return value;
}
throw new InvalidArgumentError(`invalid choice '${value}'`);
}
function positiveNumber(value: string): number {
const parsed = Number.parseInt(value, 10);
if (!Number.isInteger(parsed) || parsed <= 0) {
@ -97,7 +90,6 @@ function shouldShowSetupEntryMenu(
agents?: boolean;
target?: string;
global?: boolean;
project?: boolean;
skipAgents?: boolean;
yes?: boolean;
input?: boolean;
@ -142,7 +134,6 @@ function shouldShowSetupEntryMenu(
metabaseDatabaseId?: number;
notionCrawlMode?: string;
notionRootPageId?: string[];
skipInitialSourceIngest?: boolean;
skipSources?: boolean;
},
command: Command,
@ -172,7 +163,6 @@ function shouldShowSetupEntryMenu(
'agents',
'target',
'global',
'project',
'skipAgents',
'yes',
'input',
@ -211,7 +201,6 @@ function shouldShowSetupEntryMenu(
'sourceTarget',
'metabaseDatabaseId',
'notionCrawlMode',
'skipInitialSourceIngest',
'skipSources',
].some((optionName) => optionWasSpecified(command, optionName));
}
@ -220,9 +209,9 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo
const setup = program
.command('setup')
.description('Set up or resume a local KTX project')
.option('--project-dir <path>', 'KTX project directory')
.option('--new', 'Create a new KTX project before setup', false)
.option('--existing', 'Use an existing KTX project', false)
.addOption(new Option('--project-dir <path>', 'KTX project directory').hideHelp())
.addOption(new Option('--new', 'Create a new KTX project before setup').hideHelp().default(false))
.addOption(new Option('--existing', 'Use an existing KTX project').hideHelp().default(false))
.option('--agents', 'Install agent integration only', false)
.addOption(
new Option('--target <target>', 'Agent target').choices([
@ -233,94 +222,124 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo
'universal',
]),
)
.addOption(new Option('--agent-scope <scope>', 'Agent install scope').argParser(agentScope).default('project'))
.option('--project', 'Install agent integration into the project scope', false)
.option('--global', 'Install agent integration into the global target scope', false)
.option('--skip-agents', 'Leave agent integration incomplete for now', false)
.addOption(new Option('--skip-agents', 'Leave agent integration incomplete for now').hideHelp().default(false))
.option('--yes', 'Accept safe defaults in non-interactive setup', false)
.option('--no-input', 'Disable interactive terminal input')
.addOption(new Option('--llm-backend <backend>', 'LLM backend').argParser(llmBackend))
.option('--anthropic-api-key-env <name>', 'Environment variable containing the Anthropic API key')
.option('--anthropic-api-key-file <path>', 'File containing the Anthropic API key')
.option('--anthropic-model <model>', 'Anthropic model ID to validate and save')
.option('--vertex-project <project>', 'Google Vertex AI project ID, env:NAME, or file:/path')
.option('--vertex-location <location>', 'Google Vertex AI location, env:NAME, or file:/path')
.addOption(new Option('--skip-llm', 'Leave LLM setup incomplete for now').hideHelp().default(false))
.addOption(new Option('--embedding-backend <backend>', 'Embedding backend').argParser(embeddingBackend))
.option('--embedding-api-key-env <name>', 'Environment variable containing the embedding provider API key')
.option('--embedding-api-key-file <path>', 'File containing the embedding provider API key')
.addOption(new Option('--skip-embeddings', 'Leave embedding setup incomplete for now').hideHelp().default(false))
.option(
'--database <driver>',
'Database driver to configure; repeatable',
(value, previous: KtxSetupDatabaseDriver[]) => {
return [...previous, databaseDriver(value)];
},
[] as KtxSetupDatabaseDriver[],
)
.option(
'--database-connection-id <id>',
'Existing selected connection id or new connection id',
(value, previous: string[]) => [...previous, value],
[],
)
.option('--new-database-connection-id <id>', 'Connection id for one new database connection', (value) => {
if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(value)) {
throw new InvalidArgumentError(`Unsafe connection id: ${value}`);
}
return value;
})
.option('--database-url <url>', 'URL, env:NAME, or file:/path for one new URL-style database connection')
.option(
'--database-schema <schema>',
'Database schema to include; repeatable',
(value, previous: string[]) => [...previous, value],
[],
)
.option('--enable-historic-sql', 'Enable Historic SQL when the selected database supports it', false)
.option('--disable-historic-sql', 'Disable Historic SQL for the selected database', false)
.option('--historic-sql-window-days <number>', 'Historic SQL query-history window', positiveInteger)
.option('--historic-sql-min-executions <number>', 'Minimum Historic SQL executions for a template', positiveInteger)
.option(
'--historic-sql-service-account-pattern <pattern>',
'Historic SQL service-account regex; repeatable',
(value, previous: string[]) => [...previous, value],
[],
)
.option(
'--historic-sql-redaction-pattern <pattern>',
'Historic SQL SQL-literal redaction regex; repeatable',
(value, previous: string[]) => [...previous, value],
[],
)
.option('--skip-databases', 'Leave database setup incomplete; KTX cannot work until a primary source is added', false)
.addOption(new Option('--source <type>', 'Source connector type').argParser(sourceType))
.option('--source-connection-id <id>', 'Connection id for source setup')
.option('--source-path <path>', 'Local source path for dbt, MetricFlow, or LookML')
.option('--source-git-url <url>', 'Git URL for dbt, MetricFlow, or LookML')
.option('--source-branch <branch>', 'Git branch for source setup')
.option('--source-subpath <path>', 'Repo subpath for source setup')
.option('--source-auth-token-ref <ref>', 'env: or file: credential ref for source repo auth')
.option('--source-url <url>', 'Source service URL for Metabase or Looker')
.option('--source-api-key-ref <ref>', 'env: or file: API key ref for Metabase or Notion')
.option('--source-client-id <id>', 'Looker client id')
.option('--source-client-secret-ref <ref>', 'env: or file: Looker client secret ref')
.option('--source-warehouse-connection-id <id>', 'Mapped warehouse connection id')
.option('--source-project-name <name>', 'dbt project name override')
.option('--source-profiles-path <path>', 'dbt profiles path')
.option('--source-target <target>', 'dbt target or source-specific mapping target')
.option('--metabase-database-id <id>', 'Metabase database id to map', positiveNumber)
.addOption(new Option('--llm-backend <backend>', 'LLM backend').argParser(llmBackend).hideHelp())
.addOption(
new Option('--notion-crawl-mode <mode>', 'Notion crawl mode').choices(['all_accessible', 'selected_roots']),
new Option('--anthropic-api-key-env <name>', 'Environment variable containing the Anthropic API key').hideHelp(),
)
.option(
'--notion-root-page-id <id>',
'Notion root page id; repeatable',
(value, previous: string[]) => [...previous, value],
[],
.addOption(
new Option('--anthropic-api-key-file <path>', 'File containing the Anthropic API key').hideHelp(),
)
.option('--skip-initial-source-ingest', 'Validate source setup without building source context during setup', false)
.option('--skip-sources', 'Mark optional source setup complete with no sources', false)
.addOption(new Option('--anthropic-model <model>', 'Anthropic model ID to validate and save').hideHelp())
.addOption(new Option('--vertex-project <project>', 'Google Vertex AI project ID, env:NAME, or file:/path').hideHelp())
.addOption(new Option('--vertex-location <location>', 'Google Vertex AI location, env:NAME, or file:/path').hideHelp())
.addOption(new Option('--skip-llm', 'Leave LLM setup incomplete for now').hideHelp().default(false))
.addOption(new Option('--embedding-backend <backend>', 'Embedding backend').argParser(embeddingBackend).hideHelp())
.addOption(
new Option(
'--embedding-api-key-env <name>',
'Environment variable containing the embedding provider API key',
).hideHelp(),
)
.addOption(
new Option('--embedding-api-key-file <path>', 'File containing the embedding provider API key').hideHelp(),
)
.addOption(new Option('--skip-embeddings', 'Leave embedding setup incomplete for now').hideHelp().default(false))
.addOption(
new Option('--database <driver>', 'Database driver to configure; repeatable')
.argParser((value, previous: KtxSetupDatabaseDriver[]) => {
return [...previous, databaseDriver(value)];
})
.default([] as KtxSetupDatabaseDriver[])
.hideHelp(),
)
.addOption(
new Option('--database-connection-id <id>', 'Existing selected connection id or new connection id')
.argParser((value, previous: string[]) => [...previous, value])
.default([] as string[])
.hideHelp(),
)
.addOption(
new Option('--new-database-connection-id <id>', 'Connection id for one new database connection')
.argParser((value) => {
if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(value)) {
throw new InvalidArgumentError(`Unsafe connection id: ${value}`);
}
return value;
})
.hideHelp(),
)
.addOption(
new Option('--database-url <url>', 'URL, env:NAME, or file:/path for one new URL-style database connection').hideHelp(),
)
.addOption(
new Option('--database-schema <schema>', 'Database schema to include; repeatable')
.argParser((value, previous: string[]) => [...previous, value])
.default([] as string[])
.hideHelp(),
)
.addOption(
new Option('--enable-historic-sql', 'Enable Historic SQL when the selected database supports it')
.hideHelp()
.default(false),
)
.addOption(
new Option('--disable-historic-sql', 'Disable Historic SQL for the selected database').hideHelp().default(false),
)
.addOption(new Option('--historic-sql-window-days <number>', 'Historic SQL query-history window').argParser(positiveInteger).hideHelp())
.addOption(
new Option('--historic-sql-min-executions <number>', 'Minimum Historic SQL executions for a template')
.argParser(positiveInteger)
.hideHelp(),
)
.addOption(
new Option('--historic-sql-service-account-pattern <pattern>', 'Historic SQL service-account regex; repeatable')
.argParser((value, previous: string[]) => [...previous, value])
.default([] as string[])
.hideHelp(),
)
.addOption(
new Option('--historic-sql-redaction-pattern <pattern>', 'Historic SQL SQL-literal redaction regex; repeatable')
.argParser((value, previous: string[]) => [...previous, value])
.default([] as string[])
.hideHelp(),
)
.addOption(
new Option('--skip-databases', 'Leave database setup incomplete; KTX cannot work until a primary source is added')
.hideHelp()
.default(false),
)
.addOption(new Option('--source <type>', 'Source connector type').argParser(sourceType).hideHelp())
.addOption(new Option('--source-connection-id <id>', 'Connection id for source setup').hideHelp())
.addOption(new Option('--source-path <path>', 'Local source path for dbt, MetricFlow, or LookML').hideHelp())
.addOption(new Option('--source-git-url <url>', 'Git URL for dbt, MetricFlow, or LookML').hideHelp())
.addOption(new Option('--source-branch <branch>', 'Git branch for source setup').hideHelp())
.addOption(new Option('--source-subpath <path>', 'Repo subpath for source setup').hideHelp())
.addOption(new Option('--source-auth-token-ref <ref>', 'env: or file: credential ref for source repo auth').hideHelp())
.addOption(new Option('--source-url <url>', 'Source service URL for Metabase or Looker').hideHelp())
.addOption(new Option('--source-api-key-ref <ref>', 'env: or file: API key ref for Metabase or Notion').hideHelp())
.addOption(new Option('--source-client-id <id>', 'Looker client id').hideHelp())
.addOption(new Option('--source-client-secret-ref <ref>', 'env: or file: Looker client secret ref').hideHelp())
.addOption(new Option('--source-warehouse-connection-id <id>', 'Mapped warehouse connection id').hideHelp())
.addOption(new Option('--source-project-name <name>', 'dbt project name override').hideHelp())
.addOption(new Option('--source-profiles-path <path>', 'dbt profiles path').hideHelp())
.addOption(new Option('--source-target <target>', 'dbt target or source-specific mapping target').hideHelp())
.addOption(new Option('--metabase-database-id <id>', 'Metabase database id to map').argParser(positiveNumber).hideHelp())
.addOption(
new Option('--notion-crawl-mode <mode>', 'Notion crawl mode')
.choices(['all_accessible', 'selected_roots'])
.hideHelp(),
)
.addOption(
new Option('--notion-root-page-id <id>', 'Notion root page id; repeatable')
.argParser((value, previous: string[]) => [...previous, value])
.default([] as string[])
.hideHelp(),
)
.addOption(new Option('--skip-sources', 'Mark optional source setup complete with no sources').hideHelp().default(false))
.showHelpAfterError();
setup.hook('preAction', (_thisCommand, actionCommand) => {
@ -371,7 +390,7 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo
}
const mode = options.new ? 'new' : options.existing ? 'existing' : 'auto';
const resolvedAgentScope = options.global ? 'global' : options.agentScope;
const resolvedAgentScope = options.global ? 'global' : 'project';
await runSetupArgs(context, {
command: 'run',
projectDir: resolveCommandProjectDir(command),

5
packages/cli/src/commands/status-commands.ts
View file

@ -16,8 +16,9 @@ export function registerStatusCommands(program: Command, context: KtxCliCommandC
.command('status')
.description('Check current KTX setup and project readiness')
.option('--json', 'Print JSON output', false)
.option('-v, --verbose', 'Show every check, including passing ones', false)
.option('--no-input', 'Disable interactive terminal input')
.action(async (options: { json?: boolean; input?: boolean }, command) => {
.action(async (options: { json?: boolean; verbose?: boolean; input?: boolean }, command) => {
const runner = context.deps.doctor ?? (await import('../doctor.js')).runKtxDoctor;
const explicitOrEnvProjectDir = resolveCommandProjectDirOverride(command);
const nearestProjectDir = explicitOrEnvProjectDir ? undefined : findNearestKtxProjectDir(process.cwd());
@ -27,6 +28,7 @@ export function registerStatusCommands(program: Command, context: KtxCliCommandC
{
command: 'setup',
outputMode: outputMode(options),
verbose: options.verbose === true,
...inputMode(options),
},
context.io,
@ -40,6 +42,7 @@ export function registerStatusCommands(program: Command, context: KtxCliCommandC
command: 'project',
projectDir: resolveCommandProjectDir(command),
outputMode: outputMode(options),
verbose: options.verbose === true,
...inputMode(options),
},
context.io,

2
packages/cli/src/connection.test.ts
View file

@ -159,7 +159,7 @@ describe('runKtxConnection', () => {
prod_metabase: {
driver: 'metabase',
api_url: 'http://metabase.example.test',
api_key: 'mb_test',
api_key: 'mb_test', // pragma: allowlist secret
},
});
const testConnection = vi.fn(async () => ({ success: true as const }));

188
packages/cli/src/database-tree-picker.test.ts Normal file
View file

@ -0,0 +1,188 @@
import { describe, expect, it, vi } from 'vitest';
import {
pickDatabaseScope,
type DatabaseTreePickerRenderer,
type PickDatabaseScopeArgs,
} from './database-tree-picker.js';
import type { TreePickerChrome, TreePickerResult } from './tree-picker-tui.js';
import type { PickerState } from './tree-picker-state.js';
function makeIo() {
let stdout = '';
let stderr = '';
return {
io: {
stdout: { isTTY: true, write: (chunk: string) => { stdout += chunk; } },
stderr: { write: (chunk: string) => { stderr += chunk; } },
},
stdout: () => stdout,
stderr: () => stderr,
};
}
function captureRenderer(): {
renderer: DatabaseTreePickerRenderer;
capture: { chrome?: TreePickerChrome; state?: PickerState };
setResult: (result: TreePickerResult) => void;
} {
const capture: { chrome?: TreePickerChrome; state?: PickerState } = {};
let nextResult: TreePickerResult = { kind: 'quit' };
const renderer: DatabaseTreePickerRenderer = vi.fn(async (chrome, state) => {
capture.chrome = chrome;
capture.state = state;
return nextResult;
});
return {
renderer,
capture,
setResult: (result) => {
nextResult = result;
},
};
}
const discovered = [
{ schema: 'analytics', name: 'customers', kind: 'table' as const },
{ schema: 'analytics', name: 'orders', kind: 'table' as const },
{ schema: 'public', name: 'events', kind: 'view' as const },
{ schema: 'public', name: 'sessions', kind: 'table' as const },
];
function baseArgs(overrides: Partial<PickDatabaseScopeArgs> = {}): PickDatabaseScopeArgs {
return {
connectionId: 'warehouse',
schemaNoun: 'schema',
schemaNounPlural: 'schemas',
discovered,
existing: { enabledTables: [] },
defaultSchemas: ['analytics'],
supportsSchemaScope: true,
...overrides,
};
}
describe('pickDatabaseScope', () => {
it('builds a 2-level tree (schemas as parents, tables as children) and uses save-empty action', async () => {
const { renderer, capture, setResult } = captureRenderer();
setResult({ kind: 'quit' });
await pickDatabaseScope(baseArgs(), makeIo().io, renderer);
expect(capture.state?.skipEmptyAction).toBe('save-empty');
const schemaIds = capture.state?.tree.filter((n) => n.parentId === null).map((n) => n.id);
const tableIds = capture.state?.tree.filter((n) => n.parentId !== null).map((n) => n.id);
expect((schemaIds ?? []).sort()).toEqual(['analytics', 'public']);
expect((tableIds ?? []).sort()).toEqual([
'analytics.customers',
'analytics.orders',
'public.events',
'public.sessions',
]);
expect(capture.state?.byId.get('public.events')?.title).toBe('events (view)');
});
it('pre-checks default schemas at the parent level when no existing selection', async () => {
const { renderer, capture, setResult } = captureRenderer();
setResult({ kind: 'quit' });
await pickDatabaseScope(baseArgs({ defaultSchemas: ['analytics'] }), makeIo().io, renderer);
expect([...(capture.state?.checked ?? [])]).toEqual(['analytics']);
});
it('collapses an existing full-schema selection back into the parent check', async () => {
const { renderer, capture, setResult } = captureRenderer();
setResult({ kind: 'quit' });
await pickDatabaseScope(
baseArgs({ existing: { enabledTables: ['analytics.customers', 'analytics.orders'] } }),
makeIo().io,
renderer,
);
expect([...(capture.state?.checked ?? [])]).toEqual(['analytics']);
});
it('keeps a partial existing selection at the leaf level', async () => {
const { renderer, capture, setResult } = captureRenderer();
setResult({ kind: 'quit' });
await pickDatabaseScope(
baseArgs({ existing: { enabledTables: ['analytics.customers'] } }),
makeIo().io,
renderer,
);
expect([...(capture.state?.checked ?? [])]).toEqual(['analytics.customers']);
});
it('expands a selected schema parent into all its tables and derives activeSchemas', async () => {
const { renderer, setResult } = captureRenderer();
setResult({ kind: 'save', selectedIds: ['analytics'] });
const result = await pickDatabaseScope(baseArgs(), makeIo().io, renderer);
expect(result).toEqual({
kind: 'selected',
activeSchemas: ['analytics'],
enabledTables: ['analytics.customers', 'analytics.orders'],
});
});
it('combines parent and individual leaf selections without duplicate tables', async () => {
const { renderer, setResult } = captureRenderer();
setResult({ kind: 'save', selectedIds: ['analytics', 'public.events'] });
const result = await pickDatabaseScope(baseArgs(), makeIo().io, renderer);
expect(result).toEqual({
kind: 'selected',
activeSchemas: ['analytics', 'public'],
enabledTables: ['analytics.customers', 'analytics.orders', 'public.events'],
});
});
it('treats empty save as enable-all', async () => {
const { renderer, setResult } = captureRenderer();
setResult({ kind: 'save', selectedIds: [] });
const result = await pickDatabaseScope(baseArgs(), makeIo().io, renderer);
expect(result).toEqual({
kind: 'selected',
activeSchemas: ['analytics', 'public'],
enabledTables: [
'analytics.customers',
'analytics.orders',
'public.events',
'public.sessions',
],
});
});
it('omits activeSchemas when the driver does not support a schema scope', async () => {
const { renderer, setResult } = captureRenderer();
setResult({ kind: 'save', selectedIds: ['analytics'] });
const result = await pickDatabaseScope(
baseArgs({ supportsSchemaScope: false }),
makeIo().io,
renderer,
);
expect(result).toEqual({
kind: 'selected',
activeSchemas: [],
enabledTables: ['analytics.customers', 'analytics.orders'],
});
});
it('returns back when the picker quits', async () => {
const { renderer, setResult } = captureRenderer();
setResult({ kind: 'quit' });
const result = await pickDatabaseScope(baseArgs(), makeIo().io, renderer);
expect(result).toEqual({ kind: 'back' });
});
});

210
packages/cli/src/database-tree-picker.ts Normal file
View file

@ -0,0 +1,210 @@
import type { KtxTableListEntry } from '@ktx/context/scan';
import type { KtxCliIo } from './cli-runtime.js';
import { profileMark } from './startup-profile.js';
import {
buildInitialState,
buildPickerTree,
type PickerState,
type TreePickerNode,
type TreePickerNodeInput,
} from './tree-picker-state.js';
import {
renderTreePickerTui,
type TreePickerChrome,
type TreePickerResult,
type TreePickerTuiIo,
} from './tree-picker-tui.js';
profileMark('module:database-tree-picker');
const DATABASE_SCRIPTED_MODE_HINT =
'Database picker requires a TTY. Use --no-input and the relevant flags for scripted mode.';
export type DatabaseTreePickerRenderer = (
chrome: TreePickerChrome,
initialState: PickerState,
io: TreePickerTuiIo,
) => Promise<TreePickerResult>;
function defaultRenderer(
chrome: TreePickerChrome,
initialState: PickerState,
io: TreePickerTuiIo,
): Promise<TreePickerResult> {
return renderTreePickerTui({ chrome, initialState }, io, { scriptedModeHint: DATABASE_SCRIPTED_MODE_HINT });
}
export type DatabaseScopePickResult =
| { kind: 'selected'; activeSchemas: string[]; enabledTables: string[] }
| { kind: 'back' };
export interface PickDatabaseScopeArgs {
connectionId: string;
schemaNoun: string;
schemaNounPlural: string;
discovered: readonly KtxTableListEntry[];
existing: { enabledTables: readonly string[] };
defaultSchemas: readonly string[];
supportsSchemaScope: boolean;
}
function qualifiedTableId(entry: KtxTableListEntry): string {
return `${entry.schema}.${entry.name}`;
}
function tableTitle(entry: KtxTableListEntry): string {
return entry.kind === 'view' ? `${entry.name} (view)` : entry.name;
}
function buildTreeInputs(discovered: readonly KtxTableListEntry[]): {
inputs: TreePickerNodeInput[];
schemaIds: string[];
allTables: string[];
} {
const schemaSeen = new Set<string>();
const schemaIds: string[] = [];
for (const entry of discovered) {
if (!schemaSeen.has(entry.schema)) {
schemaSeen.add(entry.schema);
schemaIds.push(entry.schema);
}
}
const inputs: TreePickerNodeInput[] = [];
for (const schema of schemaIds) {
inputs.push({ id: schema, title: schema, archived: false, parentId: null });
}
for (const entry of discovered) {
inputs.push({
id: qualifiedTableId(entry),
title: tableTitle(entry),
archived: false,
parentId: entry.schema,
});
}
return { inputs, schemaIds, allTables: discovered.map(qualifiedTableId) };
}
function initialSelectionForExisting(
existing: readonly string[],
byId: Map<string, TreePickerNode>,
): string[] {
const tableIds = new Set(
[...byId.values()].filter((node) => node.parentId !== null).map((node) => node.id),
);
const existingTables = new Set(existing.filter((id) => tableIds.has(id)));
const schemaChildren = new Map<string, string[]>();
for (const node of byId.values()) {
if (node.parentId === null && node.childIds.length > 0) {
schemaChildren.set(node.id, [...node.childIds]);
}
}
const result: string[] = [];
for (const [schema, children] of schemaChildren) {
const allChecked = children.length > 0 && children.every((childId) => existingTables.has(childId));
if (allChecked) {
result.push(schema);
for (const childId of children) {
existingTables.delete(childId);
}
}
}
for (const id of existingTables) {
result.push(id);
}
return result;
}
function initialSelectionFromDefaults(
defaultSchemas: readonly string[],
schemaIds: readonly string[],
): string[] {
const valid = new Set(schemaIds);
const filtered = defaultSchemas.filter((s) => valid.has(s));
return filtered.length > 0 ? filtered : [...schemaIds];
}
function expandSelectedToTables(
selectedIds: readonly string[],
byId: Map<string, TreePickerNode>,
): string[] {
const expanded: string[] = [];
const seen = new Set<string>();
for (const id of selectedIds) {
const node = byId.get(id);
if (!node) continue;
if (node.childIds.length === 0) {
if (node.parentId !== null && !seen.has(id)) {
seen.add(id);
expanded.push(id);
}
continue;
}
for (const childId of node.childIds) {
if (!seen.has(childId)) {
seen.add(childId);
expanded.push(childId);
}
}
}
return expanded;
}
function schemasFromEnabledTables(enabledTables: readonly string[]): string[] {
const seen = new Set<string>();
const result: string[] = [];
for (const qualified of enabledTables) {
const schema = qualified.split('.')[0] ?? '';
if (schema.length === 0 || seen.has(schema)) continue;
seen.add(schema);
result.push(schema);
}
return result;
}
export async function pickDatabaseScope(
args: PickDatabaseScopeArgs,
io: KtxCliIo,
render: DatabaseTreePickerRenderer = defaultRenderer,
): Promise<DatabaseScopePickResult> {
const { inputs, schemaIds, allTables } = buildTreeInputs(args.discovered);
const tree = buildPickerTree(inputs);
const byId = new Map(tree.map((node) => [node.id, node]));
const tableCount = allTables.length;
const schemaCount = schemaIds.length;
const initialSelection =
args.existing.enabledTables.length > 0
? initialSelectionForExisting(args.existing.enabledTables, byId)
: initialSelectionFromDefaults(args.defaultSchemas, schemaIds);
const initialState = buildInitialState({
tree,
existingSelectedIds: initialSelection,
skipEmptyAction: 'save-empty',
});
const schemaWordPlural = schemaCount === 1 ? args.schemaNoun : args.schemaNounPlural;
const subtitleLines = [
`Connection: ${args.connectionId}`,
`Found ${tableCount} ${tableCount === 1 ? 'table' : 'tables'} across ${schemaCount} ${schemaWordPlural}.`,
`Toggle a ${args.schemaNoun} to enable all of its tables, or expand to pick individual tables.`,
];
const chrome: TreePickerChrome = {
title: `Choose tables to enable for ${args.connectionId}`,
subtitleLines,
skipEmptyMessage:
'Nothing selected. Enable all tables? Press Enter to enable all or Escape to go back.',
};
const result = await render(chrome, initialState, io as TreePickerTuiIo);
if (result.kind === 'quit') {
return { kind: 'back' };
}
const enabledTables =
result.selectedIds.length === 0 ? allTables : expandSelectedToTables(result.selectedIds, byId);
const activeSchemas = args.supportsSchemaScope ? schemasFromEnabledTables(enabledTables) : [];
return { kind: 'selected', activeSchemas, enabledTables };
}

399
packages/cli/src/doctor.test.ts
View file

@ -1,8 +1,7 @@
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import type { KtxEmbeddingConfig, KtxEmbeddingHealthCheckOptions, KtxEmbeddingHealthCheckResult } from '@ktx/llm';
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
import {
formatDoctorReport,
runKtxDoctor,
@ -31,53 +30,65 @@ function makeIo() {
};
}
type EmbeddingHealthCheck = (
config: KtxEmbeddingConfig,
options?: KtxEmbeddingHealthCheckOptions,
) => Promise<KtxEmbeddingHealthCheckResult>;
async function writeProjectConfig(projectDir: string, embeddingLines: string[]): Promise<void> {
await writeFile(
join(projectDir, 'ktx.yaml'),
[
'project: warehouse',
'connections:',
' warehouse:',
' driver: sqlite',
' path: ./warehouse.db',
'ingest:',
' adapters:',
' - live-database',
' embeddings:',
...embeddingLines.map((line) => ` ${line}`),
'',
].join('\n'),
'utf-8',
);
}
describe('formatDoctorReport', () => {
it('prints exact fixes for failing setup checks', () => {
it('shows the failing check and its fix in plain output', () => {
const checks: DoctorCheck[] = [
{ id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' },
{ id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127', group: 'toolchain' },
{
id: 'native-sqlite',
label: 'Native SQLite',
status: 'fail',
detail: 'Cannot load better-sqlite3',
fix: 'Run: pnpm run native:rebuild',
group: 'toolchain',
},
];
expect(formatDoctorReport({ title: 'KTX setup doctor', checks })).toBe(
[
'KTX setup doctor',
'PASS Node 22+: v22.16.0 ABI 127',
'FAIL Native SQLite: Cannot load better-sqlite3',
' Fix: Run: pnpm run native:rebuild',
'',
].join('\n'),
);
const output = formatDoctorReport({ title: 'KTX status', checks });
expect(output).toContain('KTX status');
expect(output).toContain('✗ Environment');
expect(output).toContain('1 of 2 need attention');
expect(output).toContain('✗ Native SQLite: Cannot load better-sqlite3');
expect(output).toContain('→ Run: pnpm run native:rebuild');
expect(output).toContain('1 issue to fix.');
});
it('lists what was checked when a group has all passing checks', () => {
const checks: DoctorCheck[] = [
{ id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0', group: 'toolchain' },
{ id: 'pnpm', label: 'pnpm 10.20+', status: 'pass', detail: '10.28.0', group: 'toolchain' },
];
const output = formatDoctorReport({ title: 'KTX status', checks });
expect(output).toContain('✓ Environment');
expect(output).toContain('Node 22+ · pnpm 10.20+');
expect(output).not.toContain('v22.16.0');
expect(output).toContain('Everything ready.');
});
it('shows the underlying detail for a single-check group on the group line', () => {
const checks: DoctorCheck[] = [
{
id: 'semantic-search-embeddings',
label: 'Semantic search embeddings',
status: 'pass',
detail: 'openai/text-embedding-3-small (1536d) probe succeeded',
group: 'search',
},
];
const output = formatDoctorReport({ title: 'KTX status', checks });
expect(output).toContain('✓ Semantic search');
expect(output).toContain('openai/text-embedding-3-small (1536d) probe succeeded');
});
it('lists every check in verbose mode', () => {
const checks: DoctorCheck[] = [
{ id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0', group: 'toolchain' },
];
const output = formatDoctorReport({ title: 'KTX status', checks }, { verbose: true });
expect(output).toContain('✓ Node 22+: v22.16.0');
});
});
@ -127,6 +138,7 @@ describe('runSetupDoctorChecks', () => {
status: 'fail',
detail: 'pnpm not found',
fix: 'Run: corepack enable && corepack prepare pnpm@10.28.0 --activate',
group: 'toolchain',
});
expect(checks).toContainEqual({
id: 'package-build',
@ -134,6 +146,7 @@ describe('runSetupDoctorChecks', () => {
status: 'fail',
detail: 'Missing packages/cli/dist/bin.js',
fix: 'Run: pnpm run build',
group: 'toolchain',
});
});
@ -154,9 +167,11 @@ describe('runSetupDoctorChecks', () => {
const testIo = makeIo();
await expect(
runKtxDoctor({ command: 'setup', outputMode: 'plain', inputMode: 'disabled' }, testIo.io, {
runSetupChecks: async () => checks,
}),
runKtxDoctor(
{ command: 'setup', outputMode: 'plain', inputMode: 'disabled', verbose: true },
testIo.io,
{ runSetupChecks: async () => checks },
),
).resolves.toBe(0);
expect(checks).toContainEqual({
@ -165,8 +180,9 @@ describe('runSetupDoctorChecks', () => {
status: 'warn',
detail: 'spawn corepack ENOENT',
fix: 'Run: corepack enable',
group: 'toolchain',
});
expect(testIo.stdout()).toContain('WARN Corepack: spawn corepack ENOENT');
expect(testIo.stdout()).toContain('⚠ Corepack: spawn corepack ENOENT');
expect(testIo.stderr()).toBe('');
});
});
@ -204,12 +220,45 @@ describe('runKtxDoctor', () => {
),
).resolves.toBe(1);
expect(testIo.stdout()).toContain('KTX setup doctor');
expect(testIo.stdout()).toContain('FAIL TypeScript package build: Missing packages/cli/dist/bin.js');
expect(testIo.stdout()).toContain('Fix: Run: pnpm run build');
expect(testIo.stdout()).toContain('KTX status');
expect(testIo.stdout()).toContain('No project here yet.');
expect(testIo.stdout()).toContain('Before you can run');
expect(testIo.stdout()).toContain('✗ TypeScript package build: Missing packages/cli/dist/bin.js');
expect(testIo.stdout()).toContain('→ Run: pnpm run build');
expect(testIo.stderr()).toBe('');
});
it('leads with `ktx setup` and hides toolchain warnings when no project exists', async () => {
const testIo = makeIo();
await expect(
runKtxDoctor(
{ command: 'setup', outputMode: 'plain', inputMode: 'disabled' },
testIo.io,
{
runSetupChecks: async () => [
{ id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0', group: 'toolchain' },
{
id: 'corepack',
label: 'Corepack',
status: 'warn',
detail: 'spawn corepack ENOENT',
fix: 'Run: corepack enable',
group: 'toolchain',
},
],
},
),
).resolves.toBe(0);
const out = testIo.stdout();
expect(out).toContain('No project here yet.');
expect(out).toContain('Run');
expect(out).toContain('ktx setup');
expect(out).not.toContain('Corepack');
expect(out).not.toContain('Node 22+');
});
it('prints JSON setup report', async () => {
const testIo = makeIo();
@ -226,12 +275,13 @@ describe('runKtxDoctor', () => {
).resolves.toBe(0);
expect(JSON.parse(testIo.stdout())).toEqual({
title: 'KTX setup doctor',
title: 'KTX status',
checks: [{ id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' }],
});
});
it('runs project checks against a valid ktx.yaml', async () => {
process.env.ANTHROPIC_API_KEY = 'test-key';
await writeFile(
join(tempDir, 'ktx.yaml'),
[
@ -240,83 +290,76 @@ describe('runKtxDoctor', () => {
' warehouse:',
' driver: sqlite',
' path: ./warehouse.db',
'llm:',
' provider:',
' backend: anthropic',
' models:',
' default: claude-sonnet-4-5',
'ingest:',
' adapters:',
' - live-database',
' embeddings:',
' backend: openai',
' model: text-embedding-3-small',
' dimensions: 1536',
'',
].join('\n'),
'utf-8',
);
process.env.OPENAI_API_KEY = 'test-key';
const testIo = makeIo();
await expect(
runKtxDoctor(
{ command: 'project', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' },
testIo.io,
{
runSetupChecks: async () => [
{ id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' },
],
},
{},
),
).resolves.toBe(0);
expect(testIo.stdout()).toContain('KTX project doctor');
expect(testIo.stdout()).toContain('PASS Project config: warehouse');
expect(testIo.stdout()).toContain('PASS Connections: 1 configured');
const out = testIo.stdout();
expect(out).toContain('KTX status');
expect(out).toContain('· warehouse');
expect(out).toContain('Connections (1)');
expect(out).toContain('LLM');
expect(out).toContain('anthropic');
expect(out).toContain('Embeddings');
expect(out).toContain('Ready.');
delete process.env.ANTHROPIC_API_KEY;
delete process.env.OPENAI_API_KEY;
});
it('includes Postgres historic-SQL readiness in project doctor output', async () => {
it('returns blocked verdict when LLM is not configured', async () => {
await writeFile(
join(tempDir, 'ktx.yaml'),
[
'project: warehouse',
'connections:',
' warehouse:',
' driver: postgres',
' url: env:WAREHOUSE_DATABASE_URL',
' historicSql:',
' enabled: true',
' dialect: postgres',
'ingest:',
' adapters:',
' - live-database',
' - historic-sql',
' driver: sqlite',
' path: ./warehouse.db',
'',
].join('\n'),
'utf-8',
);
const testIo = makeIo();
const runHistoricSqlDoctorChecks = vi.fn(async () => [
{
id: 'historic-sql-postgres-warehouse',
label: 'Postgres Historic SQL (warehouse)',
status: 'pass' as const,
detail:
'pg_stat_statements ready (PostgreSQL 16.4); info: pg_stat_statements.max is 1000; set it to at least 5000 to reduce query-template eviction churn',
},
]);
await expect(
runKtxDoctor(
{ command: 'project', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' },
testIo.io,
{
runSetupChecks: async () => [
{ id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' },
],
runHistoricSqlDoctorChecks,
},
{},
),
).resolves.toBe(0);
).resolves.toBe(1);
expect(runHistoricSqlDoctorChecks).toHaveBeenCalledTimes(1);
expect(testIo.stdout()).toContain('PASS Postgres Historic SQL (warehouse): pg_stat_statements ready');
expect(testIo.stdout()).toContain('info: pg_stat_statements.max is 1000');
expect(testIo.stdout()).not.toContain('Fix: Update the Postgres parameter group or config');
expect(testIo.stdout()).toContain('no LLM configured');
expect(testIo.stdout()).toContain('ktx setup');
});
it('warns about stale and unsupported per-driver connection fields', async () => {
process.env.ANTHROPIC_API_KEY = 'test-key';
process.env.WAREHOUSE_DATABASE_URL = 'postgresql://reader@example.test/warehouse';
process.env.NOTION_TOKEN = 'notion-secret';
await writeFile(
join(tempDir, 'ktx.yaml'),
[
@ -342,6 +385,9 @@ describe('runKtxDoctor', () => {
'ingest:',
' adapters:',
' - live-database',
'llm:',
' provider:',
' backend: anthropic',
'',
].join('\n'),
'utf-8',
@ -352,166 +398,59 @@ describe('runKtxDoctor', () => {
runKtxDoctor(
{ command: 'project', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' },
testIo.io,
{
runSetupChecks: async () => [
{ id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' },
],
runHistoricSqlDoctorChecks: async () => [],
},
{},
),
).resolves.toBe(0);
expect(testIo.stdout()).toContain('WARN Connection config (warehouse): connections.warehouse.readonly is no longer used.');
expect(testIo.stdout()).toContain(
'WARN Connection config (warehouse): connections.warehouse.historicSql.concurrency is no longer used.',
);
expect(testIo.stdout()).toContain(
'WARN Connection config (warehouse): connections.warehouse.historicSql.windowDays does not constrain pg_stat_statements.',
);
expect(testIo.stdout()).toContain('WARN Connection config (local): connections.local.file_path was removed.');
expect(testIo.stdout()).toContain(
'WARN Connection config (docs): connections.docs.last_successful_cursor is local sync state.',
);
const out = testIo.stdout();
expect(out).toContain('Warnings');
expect(out).toContain('connections.warehouse.readonly is no longer used.');
expect(out).toContain('connections.warehouse.historicSql.concurrency is no longer used.');
expect(out).toContain('connections.warehouse.historicSql.windowDays does not constrain pg_stat_statements.');
expect(out).toContain('connections.local.file_path was removed.');
expect(out).toContain('connections.docs.last_successful_cursor is local sync state.');
delete process.env.ANTHROPIC_API_KEY;
delete process.env.WAREHOUSE_DATABASE_URL;
delete process.env.NOTION_TOKEN;
});
it('warns when semantic-search embeddings are not configured', async () => {
await writeProjectConfig(tempDir, ['backend: deterministic', 'model: deterministic', 'dimensions: 8']);
process.env.ANTHROPIC_API_KEY = 'test-key';
await writeFile(
join(tempDir, 'ktx.yaml'),
[
'project: warehouse',
'connections:',
' warehouse:',
' driver: sqlite',
' path: ./warehouse.db',
'llm:',
' provider:',
' backend: anthropic',
'ingest:',
' adapters:',
' - live-database',
' embeddings:',
' backend: deterministic',
' model: deterministic',
' dimensions: 8',
'',
].join('\n'),
'utf-8',
);
const testIo = makeIo();
await expect(
runKtxDoctor(
{ command: 'project', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' },
testIo.io,
{
runSetupChecks: async () => [
{ id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' },
],
},
{},
),
).resolves.toBe(0);
expect(testIo.stdout()).toContain('WARN Semantic search embeddings: ingest.embeddings.backend is deterministic.');
expect(testIo.stdout()).toContain(
'Semantic lane will be skipped; lexical, dictionary, and token lanes remain available.',
);
expect(testIo.stdout()).toContain(
`Fix: Run: ktx setup --project-dir ${tempDir} --no-input`,
);
});
it('probes configured semantic-search embeddings for project doctor', async () => {
await writeProjectConfig(tempDir, [
'backend: sentence-transformers',
'model: all-MiniLM-L6-v2',
'dimensions: 384',
'sentenceTransformers:',
' base_url: http://127.0.0.1:8765',
" pathPrefix: ''",
]);
const healthCheck = vi.fn<EmbeddingHealthCheck>(async () => ({ ok: true }));
const testIo = makeIo();
await expect(
runKtxDoctor(
{ command: 'project', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' },
testIo.io,
{
runSetupChecks: async () => [
{ id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' },
],
embeddingHealthCheck: healthCheck,
embeddingProbeTimeoutMs: 1234,
},
),
).resolves.toBe(0);
expect(healthCheck).toHaveBeenCalledWith(
{
backend: 'sentence-transformers',
model: 'all-MiniLM-L6-v2',
dimensions: 384,
sentenceTransformers: { baseURL: 'http://127.0.0.1:8765', pathPrefix: '' },
},
{ text: 'KTX semantic search doctor probe', timeoutMs: 1234 },
);
expect(testIo.stdout()).toContain(
'PASS Semantic search embeddings: sentence-transformers/all-MiniLM-L6-v2 (384d) probe succeeded',
);
});
it('allows local sentence-transformers semantic-search probes enough time for cold start', async () => {
await writeProjectConfig(tempDir, [
'backend: sentence-transformers',
'model: all-MiniLM-L6-v2',
'dimensions: 384',
'sentenceTransformers:',
' base_url: http://127.0.0.1:8765',
" pathPrefix: ''",
]);
const healthCheck = vi.fn<EmbeddingHealthCheck>(async () => ({ ok: true }));
const testIo = makeIo();
await expect(
runKtxDoctor(
{ command: 'project', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' },
testIo.io,
{
runSetupChecks: async () => [
{ id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' },
],
embeddingHealthCheck: healthCheck,
},
),
).resolves.toBe(0);
expect(healthCheck).toHaveBeenCalledWith(
expect.objectContaining({
backend: 'sentence-transformers',
model: 'all-MiniLM-L6-v2',
dimensions: 384,
}),
{ text: 'KTX semantic search doctor probe', timeoutMs: 120_000 },
);
});
it('reports unhealthy semantic-search embeddings as a warning in JSON output', async () => {
await writeProjectConfig(tempDir, [
'backend: sentence-transformers',
'model: all-MiniLM-L6-v2',
'dimensions: 384',
'sentenceTransformers:',
' base_url: http://127.0.0.1:8765',
" pathPrefix: ''",
]);
const healthCheck = vi.fn<EmbeddingHealthCheck>(async () => ({
ok: false,
message: 'connect ECONNREFUSED 127.0.0.1:8765',
}));
const testIo = makeIo();
await expect(
runKtxDoctor(
{ command: 'project', projectDir: tempDir, outputMode: 'json', inputMode: 'disabled' },
testIo.io,
{
runSetupChecks: async () => [
{ id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' },
],
embeddingHealthCheck: healthCheck,
},
),
).resolves.toBe(0);
const report = JSON.parse(testIo.stdout()) as {
checks: Array<{ id: string; label: string; status: string; detail: string; fix?: string }>;
};
expect(report.checks).toContainEqual({
id: 'semantic-search-embeddings',
label: 'Semantic search embeddings',
status: 'warn',
detail:
'sentence-transformers/all-MiniLM-L6-v2 (384d) probe failed: connect ECONNREFUSED 127.0.0.1:8765. Semantic lane will be skipped; lexical, dictionary, and token lanes remain available.',
fix: `Run: ktx setup --project-dir ${tempDir} --no-input`,
});
expect(testIo.stdout()).toContain('Embeddings');
expect(testIo.stdout()).toContain('deterministic');
expect(testIo.stdout()).toContain('semantic search degraded');
delete process.env.ANTHROPIC_API_KEY;
});
});

542
packages/cli/src/doctor.ts
View file

@ -4,15 +4,13 @@ import { access } from 'node:fs/promises';
import { join, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { promisify } from 'node:util';
import type { KtxLocalProject, KtxProjectEmbeddingConfig } from '@ktx/context/project';
import type { KtxEmbeddingConfig, KtxEmbeddingHealthCheckOptions, KtxEmbeddingHealthCheckResult } from '@ktx/llm';
import type { HistoricSqlDoctorDeps } from './historic-sql-doctor.js';
const execFileAsync = promisify(execFile);
type DoctorStatus = 'pass' | 'warn' | 'fail';
type KtxDoctorOutputMode = 'plain' | 'json';
type KtxDoctorInputMode = 'auto' | 'disabled';
type DoctorGroup = 'toolchain' | 'project' | 'search' | 'history';
export interface DoctorCheck {
id: string;
@ -20,6 +18,7 @@ export interface DoctorCheck {
status: DoctorStatus;
detail: string;
fix?: string;
group?: DoctorGroup;
}
interface DoctorReport {
@ -28,11 +27,22 @@ interface DoctorReport {
}
export type KtxDoctorArgs =
| { command: 'setup'; outputMode: KtxDoctorOutputMode; inputMode?: KtxDoctorInputMode }
| { command: 'project'; projectDir: string; outputMode: KtxDoctorOutputMode; inputMode?: KtxDoctorInputMode };
| {
command: 'setup';
outputMode: KtxDoctorOutputMode;
inputMode?: KtxDoctorInputMode;
verbose?: boolean;
}
| {
command: 'project';
projectDir: string;
outputMode: KtxDoctorOutputMode;
inputMode?: KtxDoctorInputMode;
verbose?: boolean;
};
interface KtxDoctorIo {
stdout: { write(chunk: string): void };
stdout: { isTTY?: boolean; write(chunk: string): void };
stderr: { write(chunk: string): void };
}
@ -44,20 +54,8 @@ interface SetupDoctorDeps {
importBetterSqlite3?: () => Promise<unknown>;
}
type EmbeddingHealthCheck = (
config: KtxEmbeddingConfig,
options?: KtxEmbeddingHealthCheckOptions,
) => Promise<KtxEmbeddingHealthCheckResult>;
interface SemanticSearchDoctorDeps {
env?: NodeJS.ProcessEnv;
embeddingHealthCheck?: EmbeddingHealthCheck;
embeddingProbeTimeoutMs?: number;
}
interface KtxDoctorDeps extends SemanticSearchDoctorDeps, HistoricSqlDoctorDeps {
interface KtxDoctorDeps {
runSetupChecks?: () => Promise<DoctorCheck[]>;
runHistoricSqlDoctorChecks?: (project: KtxLocalProject, deps: HistoricSqlDoctorDeps) => Promise<DoctorCheck[]>;
}
function workspaceRootDir(): string {
@ -118,197 +116,6 @@ function check(status: DoctorStatus, id: string, label: string, detail: string,
return fix ? { id, label, status, detail, fix } : { id, label, status, detail };
}
interface ConnectionConfigWarning {
id: string;
connectionId: string;
detail: string;
fix: string;
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value !== null && !Array.isArray(value);
}
function hasOwnField(value: Record<string, unknown>, key: string): boolean {
return Object.prototype.hasOwnProperty.call(value, key);
}
function connectionConfigWarning(
connectionId: string,
key: string,
detail: string,
fix: string,
): ConnectionConfigWarning {
return {
id: `connection-config-${connectionId}-${key}`.replace(/[^a-zA-Z0-9_-]/g, '-'),
connectionId,
detail,
fix,
};
}
function connectionConfigWarnings(project: KtxLocalProject): ConnectionConfigWarning[] {
const warnings: ConnectionConfigWarning[] = [];
for (const [connectionId, connection] of Object.entries(project.config.connections)) {
const driver = String(connection.driver ?? '').toLowerCase();
if (hasOwnField(connection, 'readonly')) {
warnings.push(
connectionConfigWarning(
connectionId,
'readonly',
`connections.${connectionId}.readonly is no longer used.`,
`Remove connections.${connectionId}.readonly from ktx.yaml.`,
),
);
}
if ((driver === 'sqlite' || driver === 'sqlite3') && hasOwnField(connection, 'file_path')) {
warnings.push(
connectionConfigWarning(
connectionId,
'file-path',
`connections.${connectionId}.file_path was removed.`,
`Rename connections.${connectionId}.file_path to path.`,
),
);
}
if (driver === 'notion' && hasOwnField(connection, 'last_successful_cursor')) {
warnings.push(
connectionConfigWarning(
connectionId,
'last-successful-cursor',
`connections.${connectionId}.last_successful_cursor is local sync state.`,
'Remove it from ktx.yaml. KTX stores the Notion cursor in .ktx/db.sqlite.',
),
);
}
const historicSql = isRecord(connection.historicSql) ? connection.historicSql : null;
if (!historicSql) {
continue;
}
if (hasOwnField(historicSql, 'concurrency')) {
warnings.push(
connectionConfigWarning(
connectionId,
'historic-sql-concurrency',
`connections.${connectionId}.historicSql.concurrency is no longer used.`,
`Remove connections.${connectionId}.historicSql.concurrency from ktx.yaml.`,
),
);
}
const historicDialect = String(historicSql.dialect ?? driver).toLowerCase();
if (
(historicDialect === 'postgres' || historicDialect === 'postgresql') &&
hasOwnField(historicSql, 'windowDays')
) {
warnings.push(
connectionConfigWarning(
connectionId,
'historic-sql-window-days',
`connections.${connectionId}.historicSql.windowDays does not constrain pg_stat_statements.`,
`Remove connections.${connectionId}.historicSql.windowDays from ktx.yaml.`,
),
);
}
}
return warnings;
}
const SEMANTIC_SEARCH_HEALTH_TEXT = 'KTX semantic search doctor probe';
const SEMANTIC_SEARCH_HEALTH_TIMEOUT_MS = 5_000;
const SEMANTIC_SEARCH_LOCAL_HEALTH_TIMEOUT_MS = 120_000;
function semanticEmbeddingSetupFix(projectDir: string, backend: KtxProjectEmbeddingConfig['backend']): string {
if (backend === 'openai') {
return `Set OPENAI_API_KEY or rerun: ktx setup --project-dir ${projectDir} --embedding-backend openai --no-input`;
}
return `Run: ktx setup --project-dir ${projectDir} --no-input`;
}
function embeddingConfigLabel(config: KtxProjectEmbeddingConfig | KtxEmbeddingConfig): string {
const model = config.model?.trim() || 'model not configured';
return `${config.backend}/${model} (${config.dimensions}d)`;
}
function semanticLaneFallbackDetail(reason: string): string {
return `${reason}. Semantic lane will be skipped; lexical, dictionary, and token lanes remain available.`;
}
async function defaultEmbeddingHealthCheck(
config: KtxEmbeddingConfig,
options?: KtxEmbeddingHealthCheckOptions,
): Promise<KtxEmbeddingHealthCheckResult> {
const { runKtxEmbeddingHealthCheck } = await import('@ktx/llm');
return runKtxEmbeddingHealthCheck(config, options);
}
async function runSemanticSearchEmbeddingCheck(
config: KtxProjectEmbeddingConfig,
projectDir: string,
deps: SemanticSearchDoctorDeps = {},
): Promise<DoctorCheck> {
if (config.backend === 'none' || config.backend === 'deterministic') {
return check(
'warn',
'semantic-search-embeddings',
'Semantic search embeddings',
semanticLaneFallbackDetail(`ingest.embeddings.backend is ${config.backend}`),
semanticEmbeddingSetupFix(projectDir, config.backend),
);
}
try {
const { resolveLocalKtxEmbeddingConfig } = await import('@ktx/context');
const resolved = resolveLocalKtxEmbeddingConfig(config, deps.env ?? process.env);
if (!resolved) {
return check(
'warn',
'semantic-search-embeddings',
'Semantic search embeddings',
semanticLaneFallbackDetail(`No runtime embedding config resolved for ${embeddingConfigLabel(config)}`),
semanticEmbeddingSetupFix(projectDir, config.backend),
);
}
const healthCheck = deps.embeddingHealthCheck ?? defaultEmbeddingHealthCheck;
const timeoutMs =
deps.embeddingProbeTimeoutMs ??
(resolved.backend === 'sentence-transformers'
? SEMANTIC_SEARCH_LOCAL_HEALTH_TIMEOUT_MS
: SEMANTIC_SEARCH_HEALTH_TIMEOUT_MS);
const health = await healthCheck(resolved, {
text: SEMANTIC_SEARCH_HEALTH_TEXT,
timeoutMs,
});
if (health.ok) {
return check(
'pass',
'semantic-search-embeddings',
'Semantic search embeddings',
`${embeddingConfigLabel(resolved)} probe succeeded`,
);
}
return check(
'warn',
'semantic-search-embeddings',
'Semantic search embeddings',
semanticLaneFallbackDetail(`${embeddingConfigLabel(resolved)} probe failed: ${health.message}`),
semanticEmbeddingSetupFix(projectDir, config.backend),
);
} catch (error) {
return check(
'warn',
'semantic-search-embeddings',
'Semantic search embeddings',
semanticLaneFallbackDetail(`${embeddingConfigLabel(config)} probe failed: ${failureMessage(error)}`),
semanticEmbeddingSetupFix(projectDir, config.backend),
);
}
}
export async function runSetupDoctorChecks(deps: SetupDoctorDeps = {}): Promise<DoctorCheck[]> {
const env = deps.env ?? process.env;
const root = deps.workspaceRoot ?? workspaceRootDir();
@ -402,67 +209,231 @@ export async function runSetupDoctorChecks(deps: SetupDoctorDeps = {}): Promise<
);
}
return checks;
return checks.map((entry) => ({ ...entry, group: 'toolchain' }));
}
async function runProjectChecks(projectDir: string, deps: KtxDoctorDeps = {}): Promise<DoctorCheck[]> {
const { loadKtxProject } = await import('@ktx/context/project');
const checks: DoctorCheck[] = [];
try {
const project = await loadKtxProject({ projectDir });
checks.push(check('pass', 'project-config', 'Project config', project.config.project));
const connectionCount = Object.keys(project.config.connections).length;
checks.push(
connectionCount > 0
? check('pass', 'connections', 'Connections', `${connectionCount} configured`)
: check(
'warn',
'connections',
'Connections',
'0 configured',
'Add a connection to ktx.yaml or run `ktx setup`',
),
);
for (const warning of connectionConfigWarnings(project)) {
checks.push(
check(
'warn',
warning.id,
`Connection config (${warning.connectionId})`,
warning.detail,
warning.fix,
),
);
}
checks.push(check('pass', 'storage', 'Storage', `${project.config.storage.state}/${project.config.storage.search}`));
checks.push(check('pass', 'llm-provider', 'LLM provider', project.config.llm.provider.backend));
checks.push(await runSemanticSearchEmbeddingCheck(project.config.ingest.embeddings, projectDir, deps));
const runHistoricSqlDoctorChecks =
deps.runHistoricSqlDoctorChecks ?? (await import('./historic-sql-doctor.js')).runPostgresHistoricSqlDoctorChecks;
checks.push(...(await runHistoricSqlDoctorChecks(project, deps)));
} catch (error) {
checks.push(
check(
'fail',
'project-config',
'Project config',
failureMessage(error),
`Run: ktx init ${projectDir} --name <project-name>`,
),
);
const STATUS_SYMBOL: Record<DoctorStatus, string> = { pass: '✓', warn: '⚠', fail: '✗' };
const GROUP_ORDER: DoctorGroup[] = ['toolchain', 'project', 'search', 'history'];
const GROUP_LABEL: Record<DoctorGroup, string> = {
toolchain: 'Environment',
project: 'Project',
search: 'Semantic search',
history: 'Query history',
};
function shouldUseColor(io: KtxDoctorIo): boolean {
if (io.stdout.isTTY !== true) return false;
const env = process.env;
return !env.NO_COLOR && env.TERM !== 'dumb' && !env.CI;
}
function styleStatus(useColor: boolean, status: DoctorStatus, text: string): string {
if (!useColor) return text;
const code = status === 'pass' ? 32 : status === 'warn' ? 33 : 31;
return `\u001b[${code}m${text}\u001b[39m`;
}
function styleDim(useColor: boolean, text: string): string {
return useColor ? `\u001b[2m${text}\u001b[22m` : text;
}
function styleBold(useColor: boolean, text: string): string {
return useColor ? `\u001b[1m${text}\u001b[22m` : text;
}
function groupOf(entry: DoctorCheck): DoctorGroup {
return entry.group ?? 'project';
}
function aggregateStatus(checks: DoctorCheck[]): DoctorStatus {
if (checks.some((c) => c.status === 'fail')) return 'fail';
if (checks.some((c) => c.status === 'warn')) return 'warn';
return 'pass';
}
function abbreviateHome(filePath: string | undefined): string | undefined {
if (!filePath) return filePath;
const home = process.env.HOME;
if (home && (filePath === home || filePath.startsWith(`${home}/`))) {
return filePath === home ? '~' : `~${filePath.slice(home.length)}`;
}
return checks;
return filePath;
}
export function formatDoctorReport(report: DoctorReport): string {
const lines = [report.title];
for (const item of report.checks) {
lines.push(`${item.status.toUpperCase()} ${item.label}: ${item.detail}`);
if (item.fix) {
lines.push(` Fix: ${item.fix}`);
function groupSummaryWhenAllPass(entries: DoctorCheck[]): string {
if (entries.length === 1) {
const only = entries[0]!;
return only.detail || only.label;
}
return entries.map((c) => c.label).join(' · ');
}
interface RenderOptions {
verbose: boolean;
useColor: boolean;
durationMs?: number;
projectName?: string;
projectDir?: string;
command?: 'setup' | 'project';
}
const NEXT_STEPS_PROJECT = ['ktx scan', 'ktx wiki', 'ktx sl ask "…"'];
export function formatDoctorReport(report: DoctorReport, options: Partial<RenderOptions> = {}): string {
const opts: RenderOptions = {
verbose: options.verbose ?? false,
useColor: options.useColor ?? false,
durationMs: options.durationMs,
projectName: options.projectName,
projectDir: options.projectDir,
command: options.command,
};
return renderPlainReport(report, opts);
}
function renderSetupReport(report: DoctorReport, options: RenderOptions): string {
const { verbose, useColor } = options;
const dim = (text: string) => styleDim(useColor, text);
const bold = (text: string) => styleBold(useColor, text);
const status = (s: DoctorStatus, text: string) => styleStatus(useColor, s, text);
const symbol = (s: DoctorStatus) => status(s, STATUS_SYMBOL[s]);
const fails = report.checks.filter((c) => c.status === 'fail');
const lines: string[] = [];
lines.push(bold(report.title));
lines.push('');
lines.push(` No project here yet.`);
lines.push('');
if (fails.length > 0) {
lines.push(` Before you can run ${bold('ktx setup')}, fix this:`);
for (const entry of fails) {
lines.push(` ${symbol('fail')} ${entry.label}: ${entry.detail}`);
if (entry.fix) {
lines.push(` ${dim(`→ ${entry.fix}`)}`);
}
}
lines.push('');
} else {
lines.push(` Run ${bold('ktx setup')} to get started.`);
lines.push('');
}
if (verbose) {
lines.push(dim(' Toolchain:'));
for (const entry of report.checks) {
lines.push(` ${symbol(entry.status)} ${entry.label}: ${entry.detail}`);
if (entry.fix && entry.status !== 'pass') {
lines.push(` ${dim(`→ ${entry.fix}`)}`);
}
}
lines.push('');
}
return lines.join('\n');
}
function renderPlainReport(report: DoctorReport, options: RenderOptions): string {
if (options.command === 'setup') return renderSetupReport(report, options);
const { verbose, useColor, durationMs, projectName, projectDir } = options;
const dim = (text: string) => styleDim(useColor, text);
const bold = (text: string) => styleBold(useColor, text);
const status = (s: DoctorStatus, text: string) => styleStatus(useColor, s, text);
const symbol = (s: DoctorStatus) => status(s, STATUS_SYMBOL[s]);
const lines: string[] = [];
const titleParts: string[] = [bold(report.title)];
if (projectName) titleParts.push(projectName);
const abbreviatedDir = abbreviateHome(projectDir);
const titleLine = titleParts.join(` ${dim('·')} `);
const dirSuffix = abbreviatedDir ? ` ${dim(`(${abbreviatedDir})`)}` : '';
lines.push(`${titleLine}${dirSuffix}`);
lines.push('');
const groups = new Map<DoctorGroup, DoctorCheck[]>();
for (const entry of report.checks) {
const group = groupOf(entry);
const bucket = groups.get(group) ?? [];
bucket.push(entry);
groups.set(group, bucket);
}
const orderedGroups: DoctorGroup[] = [];
for (const g of GROUP_ORDER) {
if (groups.has(g)) orderedGroups.push(g);
}
for (const g of groups.keys()) {
if (!orderedGroups.includes(g)) orderedGroups.push(g);
}
const labelWidth = orderedGroups.reduce(
(max, g) => Math.max(max, (GROUP_LABEL[g] ?? g).length),
0,
);
for (const group of orderedGroups) {
const entries = groups.get(group) ?? [];
const head = aggregateStatus(entries);
const nonPass = entries.filter((c) => c.status !== 'pass');
const label = (GROUP_LABEL[group] ?? group).padEnd(labelWidth);
if (nonPass.length === 0) {
lines.push(` ${symbol(head)} ${label} ${dim(groupSummaryWhenAllPass(entries))}`);
if (verbose) {
for (const entry of entries) {
lines.push(` ${symbol(entry.status)} ${entry.label}: ${entry.detail}`);
}
}
continue;
}
if (entries.length === 1) {
const only = entries[0]!;
lines.push(` ${symbol(only.status)} ${label} ${only.detail}`);
if (only.fix) {
lines.push(` ${' '.repeat(2 + labelWidth + 4)}${dim(`→ ${only.fix}`)}`);
}
continue;
}
lines.push(` ${symbol(head)} ${label} ${dim(`${nonPass.length} of ${entries.length} need attention`)}`);
for (const entry of entries) {
if (entry.status === 'pass' && !verbose) continue;
lines.push(` ${symbol(entry.status)} ${entry.label}: ${entry.detail}`);
if (entry.fix) {
lines.push(` ${dim(`→ ${entry.fix}`)}`);
}
}
}
lines.push('');
const totalFail = report.checks.filter((c) => c.status === 'fail').length;
const totalWarn = report.checks.filter((c) => c.status === 'warn').length;
const durationText = durationMs !== undefined ? ` ${dim(`(${(durationMs / 1000).toFixed(2)}s)`)}` : '';
if (totalFail === 0 && totalWarn === 0) {
const hint = ` ${dim('Try:')} ${NEXT_STEPS_PROJECT.join(dim(' · '))}`;
lines.push(`${status('pass', 'Everything ready.')}${hint}${durationText}`);
} else if (totalFail === 0) {
const word = totalWarn === 1 ? 'warning' : 'warnings';
lines.push(
`${status('warn', `${totalWarn} ${word}.`)} ${dim('Run')} ktx status --verbose ${dim('for full details.')}${durationText}`,
);
} else {
const fWord = totalFail === 1 ? 'issue' : 'issues';
const warnSuffix =
totalWarn > 0
? ` ${dim('·')} ${status('warn', `${totalWarn} ${totalWarn === 1 ? 'warning' : 'warnings'}`)}`
: '';
lines.push(
`${status('fail', `${totalFail} ${fWord} to fix.`)}${warnSuffix}${durationText}`,
);
}
lines.push('');
return lines.join('\n');
}
@ -470,12 +441,12 @@ function hasFailures(report: DoctorReport): boolean {
return report.checks.some((item) => item.status === 'fail');
}
function writeReport(report: DoctorReport, outputMode: KtxDoctorOutputMode, io: KtxDoctorIo): void {
function writeReport(report: DoctorReport, outputMode: KtxDoctorOutputMode, io: KtxDoctorIo, options: RenderOptions): void {
if (outputMode === 'json') {
io.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
return;
}
io.stdout.write(formatDoctorReport(report));
io.stdout.write(renderPlainReport(report, options));
}
export async function runKtxDoctor(
@ -483,18 +454,41 @@ export async function runKtxDoctor(
io: KtxDoctorIo = process,
deps: KtxDoctorDeps = {},
): Promise<number> {
const startedAt = Date.now();
try {
const runSetupChecks = deps.runSetupChecks ?? (() => runSetupDoctorChecks());
const setupChecks = await runSetupChecks();
const report: DoctorReport =
args.command === 'setup'
? { title: 'KTX setup doctor', checks: setupChecks }
: {
title: 'KTX project doctor',
checks: [...setupChecks, ...(await runProjectChecks(args.projectDir, deps))],
};
writeReport(report, args.outputMode, io);
if (args.command === 'project') {
const { loadKtxProject } = await import('@ktx/context/project');
const { buildProjectStatus, renderProjectStatus } = await import('./status-project.js');
const project = await loadKtxProject({ projectDir: args.projectDir });
const projectStatus = buildProjectStatus(project);
const verbose = args.verbose ?? false;
const toolchainChecks = verbose ? await runSetupChecks() : undefined;
if (args.outputMode === 'json') {
io.stdout.write(`${JSON.stringify(projectStatus, null, 2)}\n`);
} else {
io.stdout.write(
renderProjectStatus(projectStatus, {
verbose,
useColor: shouldUseColor(io),
durationMs: Date.now() - startedAt,
toolchainChecks,
}),
);
}
return projectStatus.verdict === 'blocked' ? 1 : 0;
}
const setupChecks = await runSetupChecks();
const report: DoctorReport = { title: 'KTX status', checks: setupChecks };
const renderOptions: RenderOptions = {
verbose: args.verbose ?? false,
useColor: shouldUseColor(io),
durationMs: Date.now() - startedAt,
command: args.command,
};
writeReport(report, args.outputMode, io, renderOptions);
return hasFailures(report) ? 1 : 0;
} catch (error) {
io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);

202
packages/cli/src/historic-sql-doctor.test.ts
View file

@ -1,202 +0,0 @@
import { buildDefaultKtxProjectConfig, type KtxProjectConnectionConfig } from '@ktx/context/project';
import { HistoricSqlExtensionMissingError } from '@ktx/context/ingest';
import { describe, expect, it, vi } from 'vitest';
import {
runPostgresHistoricSqlDoctorChecks,
type HistoricSqlDoctorProject,
type PostgresHistoricSqlDoctorProbe,
} from './historic-sql-doctor.js';
function projectWithConnections(connections: Record<string, KtxProjectConnectionConfig>): HistoricSqlDoctorProject {
return {
projectDir: '/tmp/ktx-project',
config: {
...buildDefaultKtxProjectConfig('warehouse'),
connections,
ingest: {
...buildDefaultKtxProjectConfig('warehouse').ingest,
adapters: ['live-database', 'historic-sql'],
},
},
};
}
describe('runPostgresHistoricSqlDoctorChecks', () => {
it('passes when no Postgres historic-SQL connections are enabled', async () => {
const checks = await runPostgresHistoricSqlDoctorChecks(
projectWithConnections({
warehouse: { driver: 'sqlite', path: './warehouse.db' },
}),
{
postgresHistoricSqlProbe: vi.fn<PostgresHistoricSqlDoctorProbe>(),
},
);
expect(checks).toEqual([
{
id: 'historic-sql-postgres',
label: 'Postgres Historic SQL',
status: 'pass',
detail: 'No enabled Postgres historic-SQL connections',
},
]);
});
it('passes when the PGSS probe succeeds without warnings', async () => {
const probe = vi.fn<PostgresHistoricSqlDoctorProbe>(async () => ({
pgServerVersion: 'PostgreSQL 16.4',
warnings: [],
}));
const checks = await runPostgresHistoricSqlDoctorChecks(
projectWithConnections({
warehouse: {
driver: 'postgres',
url: 'env:WAREHOUSE_DATABASE_URL',
historicSql: { enabled: true, dialect: 'postgres' },
},
}),
{ postgresHistoricSqlProbe: probe },
);
expect(probe).toHaveBeenCalledWith({
projectDir: '/tmp/ktx-project',
connectionId: 'warehouse',
connection: {
driver: 'postgres',
url: 'env:WAREHOUSE_DATABASE_URL',
historicSql: { enabled: true, dialect: 'postgres' },
},
env: process.env,
});
expect(checks).toEqual([
{
id: 'historic-sql-postgres-warehouse',
label: 'Postgres Historic SQL (warehouse)',
status: 'pass',
detail: 'pg_stat_statements ready (PostgreSQL 16.4)',
},
]);
});
it('passes with an informational note when only pg_stat_statements.max is below the recommended floor', async () => {
const checks = await runPostgresHistoricSqlDoctorChecks(
projectWithConnections({
warehouse: {
driver: 'postgres',
url: 'env:WAREHOUSE_DATABASE_URL',
historicSql: { enabled: true, dialect: 'postgres' },
},
}),
{
postgresHistoricSqlProbe: async () => ({
pgServerVersion: 'PostgreSQL 16.4',
warnings: [],
info: [
'pg_stat_statements.max is 1000; set it to at least 5000 to reduce query-template eviction churn',
],
}),
},
);
expect(checks).toEqual([
{
id: 'historic-sql-postgres-warehouse',
label: 'Postgres Historic SQL (warehouse)',
status: 'pass',
detail:
'pg_stat_statements ready (PostgreSQL 16.4); info: pg_stat_statements.max is 1000; set it to at least 5000 to reduce query-template eviction churn',
},
]);
});
it('warns when pg_stat_statements tracking is disabled', async () => {
const checks = await runPostgresHistoricSqlDoctorChecks(
projectWithConnections({
warehouse: {
driver: 'postgres',
url: 'env:WAREHOUSE_DATABASE_URL',
historicSql: { enabled: true, dialect: 'postgres' },
},
}),
{
postgresHistoricSqlProbe: async () => ({
pgServerVersion: 'PostgreSQL 16.4',
warnings: [
'pg_stat_statements.track is none; set it to top or all in the Postgres parameter group or config',
],
info: [
'pg_stat_statements.max is 1000; set it to at least 5000 to reduce query-template eviction churn',
],
}),
},
);
expect(checks).toEqual([
{
id: 'historic-sql-postgres-warehouse',
label: 'Postgres Historic SQL (warehouse)',
status: 'warn',
detail:
'pg_stat_statements ready (PostgreSQL 16.4) with warnings: pg_stat_statements.track is none; set it to top or all in the Postgres parameter group or config; info: pg_stat_statements.max is 1000; set it to at least 5000 to reduce query-template eviction churn',
fix: 'Update the Postgres parameter group or config, then rerun `ktx status --project-dir /tmp/ktx-project`',
},
]);
});
it('fails when a connection has postgres historic SQL but is not a Postgres driver', async () => {
const checks = await runPostgresHistoricSqlDoctorChecks(
projectWithConnections({
warehouse: {
driver: 'mysql',
url: 'env:WAREHOUSE_DATABASE_URL',
historicSql: { enabled: true, dialect: 'postgres' },
},
}),
{
postgresHistoricSqlProbe: vi.fn<PostgresHistoricSqlDoctorProbe>(),
},
);
expect(checks).toEqual([
{
id: 'historic-sql-postgres-warehouse',
label: 'Postgres Historic SQL (warehouse)',
status: 'fail',
detail: 'connections.warehouse.historicSql.dialect is postgres but driver is mysql',
fix: 'Set connections.warehouse.driver to postgres or disable historicSql for this connection',
},
]);
});
it('maps PGSS capability errors to actionable failures', async () => {
const checks = await runPostgresHistoricSqlDoctorChecks(
projectWithConnections({
warehouse: {
driver: 'postgres',
url: 'env:WAREHOUSE_DATABASE_URL',
historicSql: { enabled: true, dialect: 'postgres' },
},
}),
{
postgresHistoricSqlProbe: async () => {
throw new HistoricSqlExtensionMissingError({
dialect: 'postgres',
message: 'pg_stat_statements extension is not installed in the connection database.',
remediation: 'Run CREATE EXTENSION pg_stat_statements; against the connection database.',
});
},
},
);
expect(checks).toEqual([
{
id: 'historic-sql-postgres-warehouse',
label: 'Postgres Historic SQL (warehouse)',
status: 'fail',
detail: 'pg_stat_statements extension is not installed in the connection database.',
fix: 'Run CREATE EXTENSION pg_stat_statements; against the connection database.',
},
]);
});
});

167
packages/cli/src/historic-sql-doctor.ts
View file

@ -1,167 +0,0 @@
import type { KtxProjectConfig, KtxProjectConnectionConfig } from '@ktx/context/project';
import type { DoctorCheck } from './doctor.js';
export interface HistoricSqlDoctorProject {
projectDir: string;
config: Pick<KtxProjectConfig, 'connections' | 'ingest'>;
}
export interface PostgresHistoricSqlDoctorProbeInput {
projectDir: string;
connectionId: string;
connection: KtxProjectConnectionConfig;
env: NodeJS.ProcessEnv;
}
export interface PostgresHistoricSqlDoctorProbeResult {
pgServerVersion: string;
warnings: string[];
info?: string[];
}
export type PostgresHistoricSqlDoctorProbe = (
input: PostgresHistoricSqlDoctorProbeInput,
) => Promise<PostgresHistoricSqlDoctorProbeResult>;
export interface HistoricSqlDoctorDeps {
env?: NodeJS.ProcessEnv;
postgresHistoricSqlProbe?: PostgresHistoricSqlDoctorProbe;
}
function check(status: DoctorCheck['status'], id: string, label: string, detail: string, fix?: string): DoctorCheck {
return fix ? { id, label, status, detail, fix } : { id, label, status, detail };
}
function historicSqlRecord(connection: KtxProjectConnectionConfig): Record<string, unknown> | null {
const historicSql = connection.historicSql;
return historicSql && typeof historicSql === 'object' && !Array.isArray(historicSql)
? (historicSql as Record<string, unknown>)
: null;
}
function isEnabledPostgresHistoricSql(connection: KtxProjectConnectionConfig): boolean {
const historicSql = historicSqlRecord(connection);
return historicSql?.enabled === true && historicSql.dialect === 'postgres';
}
function isPostgresDriver(connection: KtxProjectConnectionConfig): boolean {
const driver = String(connection.driver ?? '').toLowerCase();
return driver === 'postgres' || driver === 'postgresql';
}
function checkId(connectionId: string): string {
return `historic-sql-postgres-${connectionId.replace(/[^a-z0-9_-]+/gi, '-')}`;
}
function capabilityFailureFix(error: unknown, connectionId: string, projectDir: string): string {
if (error instanceof Error && error.name === 'HistoricSqlExtensionMissingError' && 'remediation' in error) {
return String(error.remediation);
}
if (error instanceof Error && error.name === 'HistoricSqlGrantsMissingError' && 'remediation' in error) {
return String(error.remediation);
}
if (error instanceof Error && error.name === 'HistoricSqlVersionUnsupportedError') {
return 'Use PostgreSQL 14 or newer, or disable historicSql for this connection';
}
return `Fix connections.${connectionId} Postgres settings, then rerun \`ktx status --project-dir ${projectDir}\``;
}
function failureDetail(error: unknown): string {
if (error instanceof Error && error.message.trim().length > 0) {
return error.message.trim().split('\n')[0] ?? error.message.trim();
}
return String(error);
}
function readinessDetail(result: PostgresHistoricSqlDoctorProbeResult): string {
const warningText = result.warnings.length > 0 ? ` with warnings: ${result.warnings.join('; ')}` : '';
const info = result.info ?? [];
const infoText = info.length > 0 ? `; info: ${info.join('; ')}` : '';
return `pg_stat_statements ready (${result.pgServerVersion})${warningText}${infoText}`;
}
async function defaultPostgresHistoricSqlProbe(
input: PostgresHistoricSqlDoctorProbeInput,
): Promise<PostgresHistoricSqlDoctorProbeResult> {
const [{ PostgresPgssReader }, { KtxPostgresHistoricSqlQueryClient, isKtxPostgresConnectionConfig }] =
await Promise.all([import('@ktx/context/ingest'), import('@ktx/connector-postgres')]);
const inputDriver = input.connection.driver ?? 'unknown';
if (!isKtxPostgresConnectionConfig(input.connection)) {
throw new Error(`Native PostgreSQL connector cannot run driver "${inputDriver}"`);
}
const client = new KtxPostgresHistoricSqlQueryClient({
connectionId: input.connectionId,
connection: input.connection,
env: input.env,
});
try {
return await new PostgresPgssReader().probe(client);
} finally {
await client.cleanup();
}
}
export async function runPostgresHistoricSqlDoctorChecks(
project: HistoricSqlDoctorProject,
deps: HistoricSqlDoctorDeps = {},
): Promise<DoctorCheck[]> {
const targets = Object.entries(project.config.connections)
.filter(([, connection]) => isEnabledPostgresHistoricSql(connection))
.sort(([left], [right]) => left.localeCompare(right));
if (targets.length === 0) {
return [
check('pass', 'historic-sql-postgres', 'Postgres Historic SQL', 'No enabled Postgres historic-SQL connections'),
];
}
const probe = deps.postgresHistoricSqlProbe ?? defaultPostgresHistoricSqlProbe;
const env = deps.env ?? process.env;
const checks: DoctorCheck[] = [];
for (const [connectionId, connection] of targets) {
const label = `Postgres Historic SQL (${connectionId})`;
if (!isPostgresDriver(connection)) {
checks.push(
check(
'fail',
checkId(connectionId),
label,
`connections.${connectionId}.historicSql.dialect is postgres but driver is ${String(connection.driver)}`,
`Set connections.${connectionId}.driver to postgres or disable historicSql for this connection`,
),
);
continue;
}
try {
const result = await probe({ projectDir: project.projectDir, connectionId, connection, env });
if (result.warnings.length > 0) {
checks.push(
check(
'warn',
checkId(connectionId),
label,
readinessDetail(result),
`Update the Postgres parameter group or config, then rerun \`ktx status --project-dir ${project.projectDir}\``,
),
);
} else {
checks.push(check('pass', checkId(connectionId), label, readinessDetail(result)));
}
} catch (error) {
checks.push(
check(
'fail',
checkId(connectionId),
label,
failureDetail(error),
capabilityFailureFix(error, connectionId, project.projectDir),
),
);
}
}
return checks;
}

76
packages/cli/src/index.test.ts
View file

@ -444,20 +444,54 @@ describe('runKtxCli', () => {
expect(io.stderr()).toContain('Choose only one runtime install mode: --yes or --no-input');
});
it('documents setup as a bare command without subcommands', async () => {
it('documents setup with only the common interactive options visible', async () => {
const testIo = makeIo();
await expect(runKtxCli(['setup', '--help'], testIo.io)).resolves.toBe(0);
expect(testIo.stdout()).toContain('Usage: ktx setup [options]');
expect(testIo.stdout()).not.toContain('Commands:');
expect(testIo.stdout()).not.toContain('setup demo');
expect(testIo.stdout()).not.toContain('setup context');
expect(testIo.stdout()).not.toContain('--skip-llm');
expect(testIo.stdout()).not.toContain('--skip-embeddings');
expect(testIo.stdout()).not.toContain('--embedding-model');
expect(testIo.stdout()).not.toContain('--embedding-dimensions');
expect(testIo.stdout()).not.toContain('--embedding-base-url');
const stdout = testIo.stdout();
expect(stdout).toContain('Usage: ktx setup [options]');
expect(stdout).toContain('--agents');
expect(stdout).toContain('--target <target>');
expect(stdout).toContain('--global');
expect(stdout).toContain('--yes');
expect(stdout).toContain('--no-input');
expect(stdout).toContain('Global Options:');
expect(stdout.match(/--project-dir <path>/g)).toHaveLength(1);
expect(stdout).not.toContain('Commands:');
expect(stdout).not.toContain('setup demo');
expect(stdout).not.toContain('setup context');
for (const hiddenFlag of [
'--new',
'--existing',
'--agent-scope',
'--skip-agents',
'--llm-backend',
'--anthropic-api-key-env',
'--vertex-project',
'--embedding-backend',
'--database ',
'--database-connection-id',
'--new-database-connection-id',
'--enable-historic-sql',
'--historic-sql-min-executions',
'--skip-databases',
'--source ',
'--source-connection-id',
'--metabase-database-id',
'--notion-root-page-id',
'--skip-initial-source-ingest',
'--skip-sources',
'--skip-llm',
'--skip-embeddings',
'--embedding-model',
'--embedding-dimensions',
'--embedding-base-url',
]) {
expect(stdout).not.toContain(hiddenFlag);
}
expect(stdout).not.toMatch(/^ --project\s/m);
expect(testIo.stderr()).toBe('');
});
@ -725,6 +759,23 @@ describe('runKtxCli', () => {
expect(setup).not.toHaveBeenCalled();
});
it('rejects removed setup options', async () => {
const setup = vi.fn(async () => 0);
const cases = [
['setup', '--project'],
['setup', '--agent-scope', 'global'],
['setup', '--skip-initial-source-ingest'],
];
for (const args of cases) {
const testIo = makeIo();
await expect(runKtxCli(['--project-dir', tempDir, ...args], testIo.io, { setup })).resolves.toBe(1);
expect(testIo.stderr()).toMatch(/unknown option|error:/i);
}
expect(setup).not.toHaveBeenCalled();
});
it('prints ingest help without invoking ingest execution', async () => {
const testIo = makeIo();
const ingest = vi.fn();
@ -961,7 +1012,7 @@ describe('runKtxCli', () => {
expect(setup).not.toHaveBeenCalled();
expect(doctor).toHaveBeenCalledWith(
{ command: 'project', projectDir: tempDir, outputMode: 'json', inputMode: 'disabled' },
{ command: 'project', projectDir: tempDir, outputMode: 'json', inputMode: 'disabled', verbose: false },
statusIo.io,
);
expect(statusIo.stderr()).toBe('');
@ -984,7 +1035,7 @@ describe('runKtxCli', () => {
await expect(runKtxCli(['status', '--json', '--no-input'], statusIo.io, { doctor })).resolves.toBe(0);
expect(doctor).toHaveBeenCalledWith(
{ command: 'setup', outputMode: 'json', inputMode: 'disabled' },
{ command: 'setup', outputMode: 'json', inputMode: 'disabled', verbose: false },
statusIo.io,
);
expect(statusIo.stderr()).toBe('');
@ -1250,7 +1301,6 @@ describe('runKtxCli', () => {
'--agents',
'--target',
'codex',
'--project',
'--no-input',
'--yes',
],

384
packages/cli/src/notion-page-picker-tui.test.tsx
View file

@ -1,384 +0,0 @@
/* @jsxImportSource react */
import { render as renderInkTest } from 'ink-testing-library';
import { act, type ReactNode } from 'react';
import { afterEach, describe, expect, it, vi } from 'vitest';
import { buildInitialState, buildPickerTree, type NotionPickerPageInput } from './notion-page-picker-tree.js';
import {
NotionPickerApp,
notionPickerCommandForInkInput,
renderNotionPickerTui,
resolveNotionPickerWidth,
sanitizeNotionPickerTuiError,
windowItems,
windowOffset,
type NotionPickerInkInstance,
type NotionPickerInkRenderOptions,
} from './notion-page-picker-tui.js';
const IDS = {
engineering: '11111111-1111-1111-1111-111111111111',
architecture: '22222222-2222-2222-2222-222222222222',
marketing: '33333333-3333-3333-3333-333333333333',
finance: '44444444-4444-4444-4444-444444444444',
ops: '55555555-5555-5555-5555-555555555555',
sales: '66666666-6666-6666-6666-666666666666',
support: '77777777-7777-7777-7777-777777777777',
product: '88888888-8888-8888-8888-888888888888',
design: '99999999-9999-9999-9999-999999999999',
};
function pages(): NotionPickerPageInput[] {
return [
{ id: IDS.engineering, title: 'Engineering Docs', archived: false, parentId: null },
{ id: IDS.architecture, title: 'Architecture', archived: false, parentId: IDS.engineering },
{ id: IDS.marketing, title: 'Marketing', archived: false, parentId: null },
];
}
function manyPages(): NotionPickerPageInput[] {
return [
{ id: IDS.engineering, title: 'Engineering Docs', archived: false, parentId: null },
{ id: IDS.architecture, title: 'Architecture', archived: false, parentId: IDS.engineering },
{ id: IDS.marketing, title: 'Marketing', archived: false, parentId: null },
{ id: IDS.finance, title: 'Finance', archived: false, parentId: null },
{ id: IDS.ops, title: 'Operations', archived: false, parentId: null },
{ id: IDS.sales, title: 'Sales', archived: false, parentId: null },
{ id: IDS.support, title: 'Support', archived: false, parentId: null },
{ id: IDS.product, title: 'Product', archived: false, parentId: null },
{ id: IDS.design, title: 'Design', archived: false, parentId: null },
];
}
function state(mode: 'all_accessible' | 'selected_roots' = 'selected_roots') {
return buildInitialState({
tree: buildPickerTree(pages()),
existingRootPageIds: [],
currentCrawlMode: mode,
});
}
async function waitForInkInput(): Promise<void> {
await new Promise((resolve) => setTimeout(resolve, 10));
}
function fakeInkInstance(): NotionPickerInkInstance {
return {
rerender: vi.fn(),
unmount: vi.fn(),
waitUntilExit: vi.fn(async () => undefined),
};
}
function normalizeFrameWrap(frame: string | undefined): string {
return frame?.replace(/\n/g, ' ') ?? '';
}
afterEach(() => {
vi.useRealTimers();
});
describe('notionPickerCommandForInkInput', () => {
it('maps browse, search, and confirm input to reducer commands', () => {
expect(notionPickerCommandForInkInput('', { downArrow: true }, state().search, null)).toBe('cursor-down');
expect(notionPickerCommandForInkInput('', { upArrow: true }, state().search, null)).toBe('cursor-up');
expect(notionPickerCommandForInkInput('', { rightArrow: true }, state().search, null)).toBe('cursor-right');
expect(notionPickerCommandForInkInput('', { leftArrow: true }, state().search, null)).toBe('cursor-left');
expect(notionPickerCommandForInkInput(' ', {}, state().search, null)).toBe('toggle-check');
expect(notionPickerCommandForInkInput('/', {}, state().search, null)).toBe('search-start');
expect(notionPickerCommandForInkInput('a', {}, state().search, null)).toBe('select-all-visible');
expect(notionPickerCommandForInkInput('n', {}, state().search, null)).toBe('select-none');
expect(notionPickerCommandForInkInput('s', {}, state().search, null)).toBe('save-request');
expect(notionPickerCommandForInkInput('q', {}, state().search, null)).toBe('quit');
expect(notionPickerCommandForInkInput('c', { ctrl: true }, state().search, null)).toBe('quit');
expect(notionPickerCommandForInkInput('x', {}, { editing: true, query: '' }, null)).toEqual({
type: 'search-input',
value: 'x',
});
expect(notionPickerCommandForInkInput('', { backspace: true }, { editing: true, query: 'x' }, null)).toBe(
'search-backspace',
);
expect(notionPickerCommandForInkInput('', { return: true }, { editing: true, query: 'x' }, null)).toBe(
'search-submit',
);
expect(notionPickerCommandForInkInput('', { escape: true }, { editing: true, query: 'x' }, null)).toBe(
'search-cancel',
);
expect(notionPickerCommandForInkInput('y', {}, state().search, 'mode-switch')).toBe('save-confirm');
expect(notionPickerCommandForInkInput('', { return: true }, state().search, 'mode-switch')).toBe('save-confirm');
expect(notionPickerCommandForInkInput('n', {}, state().search, 'mode-switch')).toBe('save-cancel');
});
});
describe('window helpers', () => {
it('centers the selected row and returns the visible slice', () => {
expect(windowOffset(20, 10, 5)).toBe(8);
expect(windowItems(['a', 'b', 'c', 'd', 'e'], 3, 3)).toEqual({ items: ['c', 'd', 'e'], offset: 2 });
});
it('clamps picker width to the design rule', () => {
expect(resolveNotionPickerWidth(200)).toBe(120);
expect(resolveNotionPickerWidth(100)).toBe(96);
expect(resolveNotionPickerWidth(50)).toBe(60);
expect(resolveNotionPickerWidth(undefined)).toBe(96);
});
});
describe('NotionPickerApp', () => {
it('renders spec banners, row glyphs, search visibility, and hint text', () => {
const initialState = {
...state('all_accessible'),
preLoadWarnings: ['1 stored root_page_ids no longer visible'],
};
const { lastFrame } = renderInkTest(
<NotionPickerApp
initialState={initialState}
connectionId="notion-main"
workspaceLabel="Design Workspace"
cappedAtCount={5000}
currentCrawlMode="all_accessible"
terminalRows={24}
terminalWidth={100}
onExit={vi.fn()}
/>,
);
const frame = lastFrame() ?? '';
expect(frame).toContain('Notion pages visible to integration "Design Workspace"');
expect(frame).toContain('5000-page cap reached - some pages not shown');
expect(frame).toContain('1 stored root_page_ids no longer visible - they will be removed if you save');
expect(frame).toContain('▸ [ ] Engineering Docs ▸ (1)');
expect(frame).toContain(' [ ] Marketing');
expect(frame).not.toContain('Search ready: -');
expect(frame).toContain('space toggle · enter expand · / search · a all · n none · s save & exit · q quit');
});
it('renders partial discovery warnings without stale-root save suffix', () => {
const initialState = {
...state(),
preLoadWarnings: ['Notion search stopped early: rate limit after first page'],
};
const { lastFrame } = renderInkTest(
<NotionPickerApp
initialState={initialState}
connectionId="notion-main"
workspaceLabel="Design Workspace"
cappedAtCount={null}
currentCrawlMode="selected_roots"
terminalRows={24}
terminalWidth={100}
onExit={vi.fn()}
/>,
);
const frame = lastFrame() ?? '';
expect(frame).toContain('Notion search stopped early: rate limit after first page');
expect(frame).not.toContain(
'Notion search stopped early: rate limit after first page - they will be removed if you save',
);
});
it('renders checked parents and locked descendants with the locked design glyphs', () => {
const initialState = {
...state(),
checked: new Set([IDS.engineering]),
expanded: new Set([IDS.engineering]),
};
const { lastFrame } = renderInkTest(
<NotionPickerApp
initialState={initialState}
connectionId="notion-main"
workspaceLabel="Design Workspace"
cappedAtCount={null}
currentCrawlMode="selected_roots"
terminalRows={24}
terminalWidth={100}
onExit={vi.fn()}
/>,
);
const frame = lastFrame() ?? '';
expect(frame).toContain('▸ [×] Engineering Docs ▾');
expect(frame).toContain(' [~] Architecture');
});
it('supports keyboard selection, all_accessible confirmation, and save callback', async () => {
const onExit = vi.fn();
const { stdin, lastFrame } = renderInkTest(
<NotionPickerApp
initialState={state('all_accessible')}
connectionId="notion-main"
workspaceLabel="Design Workspace"
cappedAtCount={null}
currentCrawlMode="all_accessible"
terminalRows={24}
terminalWidth={100}
onExit={onExit}
/>,
);
stdin.write(' ');
await waitForInkInput();
expect(lastFrame()).toContain('[×] Engineering Docs');
stdin.write('s');
await waitForInkInput();
expect(normalizeFrameWrap(lastFrame())).toContain(
'Save will switch crawl_mode all_accessible -> selected_roots and limit ingest to 1 selected page. [y] confirm [esc] back',
);
stdin.write('y');
await waitForInkInput();
expect(onExit).toHaveBeenCalledWith({ kind: 'save', rootPageIds: [IDS.engineering] });
});
it('removes transient hints after their expiry time', async () => {
vi.useFakeTimers();
const onExit = vi.fn();
const { stdin, lastFrame } = renderInkTest(
<NotionPickerApp
initialState={state()}
connectionId="notion-main"
workspaceLabel="Design Workspace"
cappedAtCount={null}
currentCrawlMode="selected_roots"
terminalRows={24}
terminalWidth={100}
onExit={onExit}
/>,
);
await act(async () => {
stdin.write('s');
await vi.advanceTimersByTimeAsync(10);
});
expect(lastFrame()).toContain('Select at least one page or press q to quit');
await act(async () => {
await vi.advanceTimersByTimeAsync(2500);
});
expect(lastFrame()).not.toContain('Select at least one page or press q to quit');
expect(onExit).not.toHaveBeenCalled();
});
it('renders row-window overflow indicators when the visible list is clipped', async () => {
const onExit = vi.fn();
const initialState = buildInitialState({
tree: buildPickerTree(manyPages()),
existingRootPageIds: [],
currentCrawlMode: 'selected_roots',
});
initialState.expanded = new Set([IDS.engineering]);
const { stdin, lastFrame } = renderInkTest(
<NotionPickerApp
initialState={initialState}
connectionId="notion-main"
workspaceLabel="Design Workspace"
cappedAtCount={null}
currentCrawlMode="selected_roots"
terminalRows={13}
terminalWidth={100}
onExit={onExit}
/>,
);
expect(lastFrame()).toContain('↓ 4 more');
stdin.write('\u001B[B');
stdin.write('\u001B[B');
stdin.write('\u001B[B');
stdin.write('\u001B[B');
await waitForInkInput();
const frame = lastFrame() ?? '';
expect(frame).toContain('↑ ');
expect(frame).toContain('↓ ');
expect(onExit).not.toHaveBeenCalled();
});
it('returns quit without saving', async () => {
const onExit = vi.fn();
const { stdin } = renderInkTest(
<NotionPickerApp
initialState={state()}
connectionId="notion-main"
workspaceLabel="Design Workspace"
cappedAtCount={null}
currentCrawlMode="selected_roots"
terminalRows={24}
terminalWidth={100}
onExit={onExit}
/>,
);
stdin.write('q');
await waitForInkInput();
expect(onExit).toHaveBeenCalledWith({ kind: 'quit' });
});
});
describe('renderNotionPickerTui', () => {
it('returns the app result from the Ink runtime', async () => {
const io = {
stdin: { isTTY: true, setRawMode: vi.fn() },
stdout: { isTTY: true, columns: 100, rows: 24, write: vi.fn() },
stderr: { write: vi.fn() },
};
const renderInk = vi.fn((_tree: ReactNode, _options: NotionPickerInkRenderOptions) => fakeInkInstance());
await expect(
renderNotionPickerTui(
{
initialState: state(),
connectionId: 'notion-main',
workspaceLabel: 'Design Workspace',
cappedAtCount: null,
currentCrawlMode: 'selected_roots',
},
io,
{ renderInk },
),
).resolves.toEqual({ kind: 'quit' });
expect(renderInk).toHaveBeenCalledOnce();
});
it('sanitizes render errors and tells the user to use no-input mode', async () => {
expect(sanitizeNotionPickerTuiError(new Error('token=secret https://api.notion.com/v1/search'))).toBe(
'[redacted] [redacted-url]',
);
});
it('falls back to quit with a scripted-mode hint when Ink cannot initialize', async () => {
let stderr = '';
const io = {
stdin: { isTTY: false, setRawMode: vi.fn() },
stdout: { isTTY: false, columns: 100, rows: 24, write: vi.fn() },
stderr: {
write(chunk: string) {
stderr += chunk;
},
},
};
await expect(
renderNotionPickerTui(
{
initialState: state(),
connectionId: 'notion-main',
workspaceLabel: 'Design Workspace',
cappedAtCount: null,
currentCrawlMode: 'selected_roots',
},
io,
{
renderInk: vi.fn(() => {
throw new Error('token=secret');
}),
},
),
).resolves.toEqual({ kind: 'quit' });
expect(stderr).toContain('Use --no-input --notion-root-page-id <UUID> for scripted mode');
expect(stderr).not.toContain('secret');
});
});

50
packages/cli/src/notion-page-picker.test.ts
View file

@ -1,4 +1,6 @@
import { describe, expect, it, vi } from 'vitest';
import type { PickerState } from './tree-picker-state.js';
import type { TreePickerChrome, TreePickerResult, TreePickerTuiIo } from './tree-picker-tui.js';
import {
discoverNotionPickerPages,
notionPickerPageFromSearchResult,
@ -6,8 +8,6 @@ import {
pickNotionRootPages,
resolveNotionWorkspaceLabel,
type NotionPickerApi,
type PickerRenderInput,
type PickerRenderResult,
} from './notion-page-picker.js';
function makeIo() {
@ -162,20 +162,27 @@ describe('Notion page picker helpers', () => {
});
});
type RenderPickerArgs = [TreePickerChrome, PickerState, TreePickerTuiIo];
describe('pickNotionRootPages', () => {
it('discovers visible pages, warns about stale roots, renders the TUI, and returns selected roots', async () => {
const api = fakeNotionApi([
notionPage(PAGE_IDS.engineering, 'Engineering'),
notionPage(PAGE_IDS.architecture, 'Architecture', PAGE_IDS.engineering),
]);
const renderPicker = vi.fn(async (input: PickerRenderInput): Promise<PickerRenderResult> => {
expect(input.connectionId).toBe('notion-main');
expect(input.workspaceLabel).toBe('Design Workspace');
expect(input.currentCrawlMode).toBe('all_accessible');
expect(input.cappedAtCount).toBeNull();
expect(input.initialState.preLoadWarnings).toEqual(['1 stored root_page_ids no longer visible']);
return { kind: 'save', rootPageIds: [PAGE_IDS.engineering] };
});
const renderPicker = vi.fn(
async (chrome: TreePickerChrome, state: PickerState): Promise<TreePickerResult> => {
expect(chrome.title).toBe('Select Notion pages to ingest');
expect(chrome.subtitleLines).toEqual(['Workspace: Design Workspace']);
expect(chrome.warningLines ?? []).toEqual([]);
expect(chrome.confirmSaveMessage).toBeTypeOf('function');
expect(state.requireConfirmOnSave).toBe(true);
expect(state.preLoadWarnings).toEqual([
'1 stored root_page_ids no longer visible - they will be removed if you save',
]);
return { kind: 'save', selectedIds: [PAGE_IDS.engineering] };
},
);
const io = makeIo();
await expect(
@ -223,7 +230,7 @@ describe('pickNotionRootPages', () => {
makeIo().io,
{
createNotionApi,
renderPicker: vi.fn(async (): Promise<PickerRenderResult> => ({ kind: 'quit' })),
renderPicker: vi.fn(async (): Promise<TreePickerResult> => ({ kind: 'quit' })),
},
),
).resolves.toEqual({ kind: 'back' });
@ -243,11 +250,13 @@ describe('pickNotionRootPages', () => {
.mockRejectedValueOnce(new Error('rate limit after first page')),
retrieveBotUser: vi.fn(async () => ({ name: 'Notion bot', bot: { workspace_name: 'Design Workspace' } })),
};
let renderInput: PickerRenderInput | undefined;
const renderPicker = vi.fn(async (input: PickerRenderInput): Promise<PickerRenderResult> => {
renderInput = input;
return { kind: 'quit' };
});
let captured: RenderPickerArgs | undefined;
const renderPicker = vi.fn(
async (chrome: TreePickerChrome, state: PickerState, io: TreePickerTuiIo): Promise<TreePickerResult> => {
captured = [chrome, state, io];
return { kind: 'quit' };
},
);
const io = makeIo();
await expect(
@ -271,11 +280,12 @@ describe('pickNotionRootPages', () => {
).resolves.toEqual({ kind: 'back' });
expect(renderPicker).toHaveBeenCalledOnce();
if (!renderInput) {
if (!captured) {
throw new Error('renderPicker was not called');
}
expect(renderInput.initialState.preLoadWarnings).toEqual(['Notion search stopped early: rate limit after first page']);
expect(renderInput.initialState.tree.map((node) => node.title)).toEqual(['Engineering']);
const [, state] = captured;
expect(state.preLoadWarnings).toEqual(['Notion search stopped early: rate limit after first page']);
expect(state.tree.map((node) => node.title)).toEqual(['Engineering']);
expect(io.stderr()).toContain('Notion search stopped early: rate limit after first page');
});
@ -300,7 +310,7 @@ describe('pickNotionRootPages', () => {
}),
retrieveBotUser: vi.fn(async () => ({ name: 'Notion bot' })),
})),
renderPicker: vi.fn(async (): Promise<PickerRenderResult> => ({ kind: 'quit' })),
renderPicker: vi.fn(async (): Promise<TreePickerResult> => ({ kind: 'quit' })),
},
),
).resolves.toEqual({ kind: 'unavailable', message: 'Notion API unavailable' });

103
packages/cli/src/notion-page-picker.ts
View file

@ -3,13 +3,19 @@ import { type NotionApi, type NotionBotInfo, NotionClient } from '@ktx/context/i
import type { KtxProjectConnectionConfig } from '@ktx/context/project';
import type { KtxCliIo } from './cli-runtime.js';
import { profileMark } from './startup-profile.js';
import { buildInitialState, buildPickerTree, type NotionPickerPageInput } from './notion-page-picker-tree.js';
import {
type NotionPickerTuiIo,
type PickerRenderInput,
type PickerRenderResult,
renderNotionPickerTui,
} from './notion-page-picker-tui.js';
buildInitialState,
buildPickerTree,
flattenSelection,
type PickerState,
type TreePickerNodeInput,
} from './tree-picker-state.js';
import {
renderTreePickerTui,
type TreePickerChrome,
type TreePickerResult,
type TreePickerTuiIo,
} from './tree-picker-tui.js';
profileMark('module:notion-page-picker');
@ -19,8 +25,6 @@ export interface PickNotionRootPagesArgs {
}
export type NotionPickerApi = Pick<NotionApi, 'search' | 'retrieveBotUser'>;
export type { PickerRenderInput, PickerRenderResult };
export type NotionRootPagePickResult =
| { kind: 'selected'; rootPageIds: string[] }
| { kind: 'back' }
@ -29,10 +33,16 @@ export type NotionRootPagePickResult =
export interface NotionRootPagePickerDeps {
env?: Record<string, string | undefined>;
createNotionApi?: (authToken: string) => NotionPickerApi;
renderPicker?: (input: PickerRenderInput, io: NotionPickerTuiIo) => Promise<PickerRenderResult>;
renderPicker?: (
chrome: TreePickerChrome,
initialState: PickerState,
io: TreePickerTuiIo,
) => Promise<TreePickerResult>;
}
const NOTION_PICKER_PAGE_CAP = 5000;
const NOTION_SCRIPTED_MODE_HINT =
'Notion picker requires a TTY. Use --no-input --notion-root-page-id <UUID> for scripted mode.';
function assertSafeNotionPickerConnectionId(connectionId: string): void {
if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(connectionId)) {
@ -50,6 +60,14 @@ export function normalizeNotionPageId(value: string): string {
return `${lower.slice(0, 8)}-${lower.slice(8, 12)}-${lower.slice(12, 16)}-${lower.slice(16, 20)}-${lower.slice(20)}`;
}
function tryNormalizeNotionPageId(value: string): string | null {
try {
return normalizeNotionPageId(value);
} catch {
return null;
}
}
function recordValue(value: unknown): Record<string, unknown> | null {
return typeof value === 'object' && value !== null && !Array.isArray(value)
? (value as Record<string, unknown>)
@ -88,7 +106,7 @@ function extractParentPageId(page: Record<string, unknown>): string | null {
return normalizeNotionPageId(parent.page_id);
}
export function notionPickerPageFromSearchResult(result: Record<string, unknown>): NotionPickerPageInput {
export function notionPickerPageFromSearchResult(result: Record<string, unknown>): TreePickerNodeInput {
const id = typeof result.id === 'string' ? normalizeNotionPageId(result.id) : '';
if (!id) {
throw new Error('Notion page search result is missing id');
@ -104,9 +122,9 @@ export function notionPickerPageFromSearchResult(result: Record<string, unknown>
export async function discoverNotionPickerPages(
api: NotionPickerApi,
options: { cap?: number } = {},
): Promise<{ pages: NotionPickerPageInput[]; cappedAtCount: number | null; warnings: string[] }> {
): Promise<{ pages: TreePickerNodeInput[]; cappedAtCount: number | null; warnings: string[] }> {
const cap = options.cap ?? NOTION_PICKER_PAGE_CAP;
const pages: NotionPickerPageInput[] = [];
const pages: TreePickerNodeInput[] = [];
const warnings: string[] = [];
let cursor: string | null | undefined = null;
@ -171,6 +189,33 @@ function notionCrawlMode(connection: KtxProjectConnectionConfig): 'all_accessibl
return connection.crawl_mode === 'all_accessible' ? 'all_accessible' : 'selected_roots';
}
function selectedPageCountText(count: number): string {
return `${count} selected ${count === 1 ? 'page' : 'pages'}`;
}
function notionChrome(args: {
workspaceLabel: string;
cappedAtCount: number | null;
currentCrawlMode: 'all_accessible' | 'selected_roots';
}): TreePickerChrome {
const warningLines: string[] = [];
if (args.cappedAtCount) {
warningLines.push(`${args.cappedAtCount}-page cap reached - some pages not shown`);
}
return {
title: 'Select Notion pages to ingest',
subtitleLines: [`Workspace: ${args.workspaceLabel}`],
warningLines,
confirmSaveMessage:
args.currentCrawlMode === 'all_accessible'
? (state) =>
`Switch crawl_mode from all_accessible to selected_roots? Will limit ingest to ${selectedPageCountText(
flattenSelection(state.checked, state.byId).length,
)}. Press Enter to confirm or Escape to go back.`
: undefined,
};
}
export async function pickNotionRootPages(
args: PickNotionRootPagesArgs,
io: KtxCliIo = process,
@ -190,10 +235,14 @@ export async function pickNotionRootPages(
const api = deps.createNotionApi ? deps.createNotionApi(authToken) : new NotionClient(authToken);
const discovery = await discoverNotionPickerPages(api);
const tree = buildPickerTree(discovery.pages);
const normalizedExistingIds = stringArray(args.connection.root_page_ids)
.map((raw) => tryNormalizeNotionPageId(raw))
.filter((id): id is string => id !== null);
const initialState = buildInitialState({
tree,
existingRootPageIds: stringArray(args.connection.root_page_ids),
currentCrawlMode: crawlMode,
existingSelectedIds: normalizedExistingIds,
requireConfirmOnSave: crawlMode === 'all_accessible',
staleWarning: (count) => `${count} stored root_page_ids no longer visible - they will be removed if you save`,
});
const preLoadWarnings = [...discovery.warnings, ...initialState.preLoadWarnings];
const renderState =
@ -207,23 +256,25 @@ export async function pickNotionRootPages(
io.stderr.write(`${warning}\n`);
}
const workspaceLabel = await resolveNotionWorkspaceLabel(api, args.connectionId);
const result = await (deps.renderPicker ?? renderNotionPickerTui)(
{
initialState: renderState,
connectionId: args.connectionId,
workspaceLabel,
cappedAtCount: discovery.cappedAtCount,
currentCrawlMode: crawlMode,
},
io as NotionPickerTuiIo,
);
const chrome = notionChrome({
workspaceLabel,
cappedAtCount: discovery.cappedAtCount,
currentCrawlMode: crawlMode,
});
const renderPicker =
deps.renderPicker ??
((chromeArg, state, ioArg) =>
renderTreePickerTui({ chrome: chromeArg, initialState: state }, ioArg, {
scriptedModeHint: NOTION_SCRIPTED_MODE_HINT,
}));
const result = await renderPicker(chrome, renderState, io as TreePickerTuiIo);
if (result.kind === 'quit') {
return { kind: 'back' };
}
if (result.rootPageIds.length === 0) {
if (result.selectedIds.length === 0) {
return { kind: 'unavailable', message: 'Notion picker did not return any selected pages.' };
}
return { kind: 'selected', rootPageIds: result.rootPageIds };
return { kind: 'selected', rootPageIds: result.selectedIds };
} catch (error) {
return { kind: 'unavailable', message: error instanceof Error ? error.message : String(error) };
}

2
packages/cli/src/setup-agents.ts
View file

@ -82,7 +82,7 @@ export function plannedKtxAgentFiles(input: {
{ kind: 'file', path: join(codexHome, 'instructions/ktx.md'), role: 'rule' as const },
];
}
throw new Error(`Global ${input.target} installation is not supported; use --project.`);
throw new Error(`Global ${input.target} installation is not supported; omit --global.`);
}
const root = resolve(input.projectDir);

498
packages/cli/src/setup-databases.test.ts
View file

@ -5,10 +5,15 @@ import { initKtxProject, parseKtxProjectConfig, readKtxSetupState, writeKtxSetup
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import {
type KtxSetupDatabaseDriver,
type KtxSetupDatabasesDeps,
type KtxSetupDatabasesPromptAdapter,
runKtxSetupDatabasesStep,
} from './setup-databases.js';
import type { KtxCliIo } from './cli-runtime.js';
import type {
DatabaseScopePickResult,
PickDatabaseScopeArgs,
} from './database-tree-picker.js';
function makeIo() {
let stdout = '';
@ -32,6 +37,43 @@ function makeIo() {
};
}
type ScopePick =
| 'back'
| 'enable-all'
| { schemas: string[]; tables: string[] };
interface PickerStubs {
pickDatabaseScope: KtxSetupDatabasesDeps['pickDatabaseScope'];
scopeCalls: PickDatabaseScopeArgs[];
}
function makePickerStubs(options: { scopes?: ScopePick[] } = {}): PickerStubs {
const queue: ScopePick[] = [...(options.scopes ?? [])];
const scopeCalls: PickDatabaseScopeArgs[] = [];
return {
scopeCalls,
pickDatabaseScope: vi.fn(async (args: PickDatabaseScopeArgs): Promise<DatabaseScopePickResult> => {
scopeCalls.push(args);
const next = queue.shift();
if (next === undefined || next === 'enable-all') {
const enabledTables = args.discovered.map((t) => `${t.schema}.${t.name}`);
const activeSchemas = args.supportsSchemaScope
? Array.from(new Set(args.discovered.map((t) => t.schema)))
: [];
return { kind: 'selected', activeSchemas, enabledTables };
}
if (next === 'back') {
return { kind: 'back' };
}
return {
kind: 'selected',
activeSchemas: args.supportsSchemaScope ? next.schemas : [],
enabledTables: next.tables,
};
}),
};
}
function makePromptAdapter(options: {
multiselectValues?: string[][];
selectValues?: string[];
@ -240,8 +282,9 @@ describe('setup databases step', () => {
expect(prompts.select).toHaveBeenCalledWith({
message: 'Configure PostgreSQL',
options: [
{ value: 'existing:warehouse', label: 'Use existing PostgreSQL connection: warehouse' },
{ value: 'new', label: 'Add new PostgreSQL connection' },
{ value: 'existing:warehouse', label: 'Keep existing PostgreSQL connection: warehouse' },
{ value: 'edit:warehouse', label: 'Edit PostgreSQL connection: warehouse' },
{ value: 'new', label: 'Add another PostgreSQL connection' },
{ value: 'back', label: 'Back' },
],
});
@ -564,7 +607,8 @@ describe('setup databases step', () => {
message: 'Primary sources already configured: warehouse\nWhat would you like to do?',
options: [
{ value: 'continue', label: 'Continue to knowledge sources' },
{ value: 'add', label: 'Add another primary source' },
{ value: 'edit', label: 'Edit an existing primary source' },
{ value: 'add', label: 'Add additional primary sources' },
],
});
expect(testConnection).not.toHaveBeenCalled();
@ -608,11 +652,16 @@ describe('setup databases step', () => {
connectionIds: ['warehouse', 'mysql-warehouse'],
});
expect(prompts.multiselect).toHaveBeenCalledTimes(1);
expect(prompts.multiselect).toHaveBeenCalledWith(expect.objectContaining({
initialValues: ['postgres'],
required: true,
}));
expect(prompts.select).toHaveBeenCalledWith({
message: 'Primary sources already configured: warehouse\nWhat would you like to do?',
options: [
{ value: 'continue', label: 'Continue to knowledge sources' },
{ value: 'add', label: 'Add another primary source' },
{ value: 'edit', label: 'Edit an existing primary source' },
{ value: 'add', label: 'Add additional primary sources' },
],
});
expect(testConnection).toHaveBeenCalledTimes(1);
@ -642,11 +691,16 @@ describe('setup databases step', () => {
connectionIds: ['postgres-warehouse', 'mysql-warehouse'],
});
expect(prompts.multiselect).toHaveBeenCalledTimes(2);
expect(prompts.multiselect).toHaveBeenNthCalledWith(2, expect.objectContaining({
initialValues: ['postgres'],
required: true,
}));
expect(prompts.select).toHaveBeenCalledWith({
message: 'Primary sources already configured: postgres-warehouse\nWhat would you like to do?',
options: [
{ value: 'continue', label: 'Continue to knowledge sources' },
{ value: 'add', label: 'Add another primary source' },
{ value: 'edit', label: 'Edit an existing primary source' },
{ value: 'add', label: 'Add additional primary sources' },
],
});
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
@ -675,12 +729,17 @@ describe('setup databases step', () => {
connectionIds: ['postgres-warehouse'],
});
expect(prompts.multiselect).toHaveBeenCalledTimes(2);
expect(prompts.multiselect).toHaveBeenNthCalledWith(2, expect.objectContaining({
initialValues: ['postgres'],
required: true,
}));
expect(io.stdout()).not.toContain('KTX cannot work without at least one primary source');
expect(prompts.select).toHaveBeenNthCalledWith(2, {
message: 'Primary sources already configured: postgres-warehouse\nWhat would you like to do?',
options: [
{ value: 'continue', label: 'Continue to knowledge sources' },
{ value: 'add', label: 'Add another primary source' },
{ value: 'edit', label: 'Edit an existing primary source' },
{ value: 'add', label: 'Add additional primary sources' },
],
});
});
@ -715,16 +774,403 @@ describe('setup databases step', () => {
);
expect(result).toEqual({ status: 'ready', projectDir: tempDir, connectionIds: ['warehouse'] });
expect(prompts.multiselect).toHaveBeenCalledWith(expect.objectContaining({
initialValues: ['postgres'],
required: true,
}));
expect(io.stdout()).not.toContain('KTX cannot work without at least one primary source');
expect(prompts.select).toHaveBeenNthCalledWith(2, {
message: 'Primary sources already configured: warehouse\nWhat would you like to do?',
options: [
{ value: 'continue', label: 'Continue to knowledge sources' },
{ value: 'add', label: 'Add another primary source' },
{ value: 'edit', label: 'Edit an existing primary source' },
{ value: 'add', label: 'Add additional primary sources' },
],
});
});
it('returns from primary source edit selection back to the configured source menu', async () => {
await writeFile(
join(tempDir, 'ktx.yaml'),
[
'project: warehouse',
'connections:',
' warehouse:',
' driver: postgres',
' url: env:DATABASE_URL',
'setup:',
' database_connection_ids:',
' - warehouse',
'',
].join('\n'),
'utf-8',
);
await writeKtxSetupState(tempDir, { completed_steps: ['databases'] });
const prompts = makePromptAdapter({
selectValues: ['edit', 'back', 'continue'],
});
const testConnection = vi.fn(async () => 0);
const scanConnection = vi.fn(async () => 0);
const result = await runKtxSetupDatabasesStep(
{ projectDir: tempDir, inputMode: 'auto', skipDatabases: false, databaseSchemas: [] },
makeIo().io,
{ prompts, testConnection, scanConnection },
);
expect(result).toEqual({ status: 'ready', projectDir: tempDir, connectionIds: ['warehouse'] });
expect(prompts.select).toHaveBeenNthCalledWith(2, {
message: 'Primary source to edit',
options: [
{ value: 'warehouse', label: 'warehouse (PostgreSQL)' },
{ value: 'back', label: 'Back' },
],
});
expect(prompts.select).toHaveBeenNthCalledWith(3, {
message: 'Primary sources already configured: warehouse\nWhat would you like to do?',
options: [
{ value: 'continue', label: 'Continue to knowledge sources' },
{ value: 'edit', label: 'Edit an existing primary source' },
{ value: 'add', label: 'Add additional primary sources' },
],
});
expect(testConnection).not.toHaveBeenCalled();
expect(scanConnection).not.toHaveBeenCalled();
});
it('reruns table selection after editing schema scope so stale enabled tables are removed', async () => {
await writeFile(
join(tempDir, 'ktx.yaml'),
[
'project: warehouse',
'connections:',
' warehouse:',
' driver: postgres',
' url: env:DATABASE_URL',
' schemas:',
' - public',
' enabled_tables:',
' - public.orders',
'setup:',
' database_connection_ids:',
' - warehouse',
'',
].join('\n'),
'utf-8',
);
await writeKtxSetupState(tempDir, { completed_steps: ['databases'] });
const prompts = makePromptAdapter({
textValues: ['env:DATABASE_URL'],
});
let primaryMenuCount = 0;
vi.mocked(prompts.select).mockImplementation(async (options) => {
if (options.message === 'Primary sources already configured: warehouse\nWhat would you like to do?') {
primaryMenuCount += 1;
return primaryMenuCount === 1 ? 'edit' : 'continue';
}
if (options.message === 'Primary source to edit') return 'warehouse';
if (options.message === 'How do you want to connect to PostgreSQL?') return 'url';
return 'back';
});
const testConnection = vi.fn(async () => 0);
const scanConnection = vi.fn(async () => 0);
const listSchemas = vi.fn(async () => ['analytics', 'public']);
const listTables = vi.fn(async () => [{ schema: 'analytics', name: 'customers', kind: 'table' as const }]);
const pickers = makePickerStubs({
scopes: [{ schemas: ['analytics'], tables: ['analytics.customers'] }],
});
const result = await runKtxSetupDatabasesStep(
{ projectDir: tempDir, inputMode: 'auto', skipDatabases: false, databaseSchemas: [] },
makeIo().io,
{
prompts,
testConnection,
scanConnection,
listSchemas,
listTables,
pickDatabaseScope: pickers.pickDatabaseScope,
},
);
expect(result).toEqual({ status: 'ready', projectDir: tempDir, connectionIds: ['warehouse'] });
expect(prompts.text).toHaveBeenCalledWith({
message: textInputPrompt('PostgreSQL connection URL'),
placeholder: 'env:DATABASE_URL',
initialValue: 'env:DATABASE_URL',
});
expect(listTables).toHaveBeenCalledWith(tempDir, 'warehouse', ['analytics', 'public']);
expect(testConnection).toHaveBeenCalledWith(tempDir, 'warehouse', expect.anything());
expect(scanConnection).toHaveBeenCalledWith(tempDir, 'warehouse', expect.anything());
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
expect(config.connections.warehouse).toMatchObject({
schemas: ['analytics'],
enabled_tables: ['analytics.customers'],
});
});
it('preselects existing schema and table choices when editing a primary source', async () => {
await writeFile(
join(tempDir, 'ktx.yaml'),
[
'project: warehouse',
'connections:',
' warehouse:',
' driver: postgres',
' url: env:DATABASE_URL',
' schemas:',
' - public',
' enabled_tables:',
' - public.customers',
' - public.orders',
'setup:',
' database_connection_ids:',
' - warehouse',
'',
].join('\n'),
'utf-8',
);
await writeKtxSetupState(tempDir, { completed_steps: ['databases'] });
const prompts = makePromptAdapter({
textValues: ['env:DATABASE_URL'],
});
let primaryMenuCount = 0;
vi.mocked(prompts.select).mockImplementation(async (options) => {
if (options.message === 'Primary sources already configured: warehouse\nWhat would you like to do?') {
primaryMenuCount += 1;
return primaryMenuCount === 1 ? 'edit' : 'continue';
}
if (options.message === 'Primary source to edit') return 'warehouse';
if (options.message === 'How do you want to connect to PostgreSQL?') return 'url';
return 'back';
});
const listSchemas = vi.fn(async () => ['orbit_analytics', 'orbit_raw', 'public']);
const listTables = vi.fn(async () => [
{ schema: 'public', name: 'customers', kind: 'table' as const },
{ schema: 'public', name: 'orders', kind: 'table' as const },
{ schema: 'public', name: 'products', kind: 'table' as const },
]);
const pickers = makePickerStubs({
scopes: [{ schemas: ['public'], tables: ['public.customers', 'public.orders'] }],
});
const result = await runKtxSetupDatabasesStep(
{ projectDir: tempDir, inputMode: 'auto', skipDatabases: false, databaseSchemas: [] },
makeIo().io,
{
prompts,
testConnection: vi.fn(async () => 0),
scanConnection: vi.fn(async () => 0),
listSchemas,
listTables,
pickDatabaseScope: pickers.pickDatabaseScope,
},
);
expect(result).toEqual({ status: 'ready', projectDir: tempDir, connectionIds: ['warehouse'] });
expect(pickers.scopeCalls).toHaveLength(1);
expect(pickers.scopeCalls[0]).toMatchObject({
connectionId: 'warehouse',
schemaNoun: 'schema',
supportsSchemaScope: true,
existing: { enabledTables: ['public.customers', 'public.orders'] },
});
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
expect(config.connections.warehouse).toMatchObject({
schemas: ['public'],
enabled_tables: ['public.customers', 'public.orders'],
});
});
it('returns to the configured primary menu when backing out of schema review during edit', async () => {
await writeFile(
join(tempDir, 'ktx.yaml'),
[
'project: warehouse',
'connections:',
' warehouse:',
' driver: postgres',
' url: env:DATABASE_URL',
' schemas:',
' - public',
' enabled_tables:',
' - public.orders',
'setup:',
' database_connection_ids:',
' - warehouse',
'',
].join('\n'),
'utf-8',
);
await writeKtxSetupState(tempDir, { completed_steps: ['databases'] });
const prompts = makePromptAdapter({
textValues: ['env:DATABASE_URL'],
});
let primaryMenuCount = 0;
vi.mocked(prompts.select).mockImplementation(async (options) => {
if (options.message === 'Primary sources already configured: warehouse\nWhat would you like to do?') {
primaryMenuCount += 1;
return primaryMenuCount === 1 ? 'edit' : 'continue';
}
if (options.message === 'Primary source to edit') return 'warehouse';
if (options.message === 'How do you want to connect to PostgreSQL?') return 'url';
return 'back';
});
const testConnection = vi.fn(async () => 0);
const scanConnection = vi.fn(async () => 0);
const listSchemas = vi.fn(async () => ['analytics', 'public']);
const listTables = vi.fn(async () => [
{ schema: 'analytics', name: 'customers', kind: 'table' as const },
{ schema: 'public', name: 'orders', kind: 'table' as const },
]);
const pickers = makePickerStubs({ scopes: ['back'] });
const result = await runKtxSetupDatabasesStep(
{ projectDir: tempDir, inputMode: 'auto', skipDatabases: false, databaseSchemas: [] },
makeIo().io,
{
prompts,
testConnection,
scanConnection,
listSchemas,
listTables,
pickDatabaseScope: pickers.pickDatabaseScope,
},
);
expect(result).toEqual({ status: 'ready', projectDir: tempDir, connectionIds: ['warehouse'] });
expect(primaryMenuCount).toBe(2);
expect(testConnection).toHaveBeenCalledWith(tempDir, 'warehouse', expect.anything());
expect(scanConnection).not.toHaveBeenCalled();
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
expect(config.connections.warehouse).toMatchObject({
url: 'env:DATABASE_URL',
schemas: ['public'],
enabled_tables: ['public.orders'],
});
});
it('returns to the configured primary menu when backing out of table review during edit', async () => {
await writeFile(
join(tempDir, 'ktx.yaml'),
[
'project: warehouse',
'connections:',
' warehouse:',
' driver: postgres',
' url: env:DATABASE_URL',
' schemas:',
' - public',
' enabled_tables:',
' - public.orders',
'setup:',
' database_connection_ids:',
' - warehouse',
'',
].join('\n'),
'utf-8',
);
await writeKtxSetupState(tempDir, { completed_steps: ['databases'] });
const prompts = makePromptAdapter({ textValues: ['env:DATABASE_URL'] });
let primaryMenuCount = 0;
vi.mocked(prompts.select).mockImplementation(async (options) => {
if (options.message === 'Primary sources already configured: warehouse\nWhat would you like to do?') {
primaryMenuCount += 1;
return primaryMenuCount === 1 ? 'edit' : 'continue';
}
if (options.message === 'Primary source to edit') return 'warehouse';
if (options.message === 'How do you want to connect to PostgreSQL?') return 'url';
return 'back';
});
const testConnection = vi.fn(async () => 0);
const scanConnection = vi.fn(async () => 0);
const listSchemas = vi.fn(async () => ['public']);
const listTables = vi.fn(async () => [
{ schema: 'public', name: 'customers', kind: 'table' as const },
{ schema: 'public', name: 'orders', kind: 'table' as const },
]);
const pickers = makePickerStubs({ scopes: ['back'] });
const result = await runKtxSetupDatabasesStep(
{ projectDir: tempDir, inputMode: 'auto', skipDatabases: false, databaseSchemas: [] },
makeIo().io,
{
prompts,
testConnection,
scanConnection,
listSchemas,
listTables,
pickDatabaseScope: pickers.pickDatabaseScope,
},
);
expect(result).toEqual({ status: 'ready', projectDir: tempDir, connectionIds: ['warehouse'] });
expect(primaryMenuCount).toBe(2);
expect(listTables).toHaveBeenCalledWith(tempDir, 'warehouse', ['public']);
expect(scanConnection).not.toHaveBeenCalled();
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
expect(config.connections.warehouse).toMatchObject({
url: 'env:DATABASE_URL',
schemas: ['public'],
enabled_tables: ['public.orders'],
});
});
it('restores an existing primary source edit when the follow-up scan fails', async () => {
await writeFile(
join(tempDir, 'ktx.yaml'),
[
'project: warehouse',
'connections:',
' warehouse:',
' driver: postgres',
' url: env:DATABASE_URL',
' schemas:',
' - public',
' enabled_tables:',
' - public.orders',
'setup:',
' database_connection_ids:',
' - warehouse',
'',
].join('\n'),
'utf-8',
);
await writeKtxSetupState(tempDir, { completed_steps: ['databases'] });
const prompts = makePromptAdapter({
textValues: ['env:DATABASE_URL'],
});
vi.mocked(prompts.select).mockImplementation(async (options) => {
if (options.message === 'Primary sources already configured: warehouse\nWhat would you like to do?') return 'edit';
if (options.message === 'Primary source to edit') return 'warehouse';
if (options.message === 'How do you want to connect to PostgreSQL?') return 'url';
return 'back';
});
const listTables = vi.fn(async () => [
{ schema: 'public', name: 'customers', kind: 'table' as const },
{ schema: 'public', name: 'orders', kind: 'table' as const },
]);
const pickers = makePickerStubs({ scopes: ['enable-all'] });
const result = await runKtxSetupDatabasesStep(
{ projectDir: tempDir, inputMode: 'auto', skipDatabases: false, databaseSchemas: [] },
makeIo().io,
{
prompts,
testConnection: vi.fn(async () => 0),
scanConnection: vi.fn(async () => 1),
listTables,
pickDatabaseScope: pickers.pickDatabaseScope,
},
);
expect(result).toEqual({ status: 'failed', projectDir: tempDir });
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
expect(config.connections.warehouse).toMatchObject({
enabled_tables: ['public.orders'],
});
});
it('lets Escape from connection fields return to connection method selection', async () => {
const prompts = makePromptAdapter({
selectValues: ['fields', 'url'],
@ -1000,7 +1446,6 @@ describe('setup databases step', () => {
const prompts = makePromptAdapter({
selectValues: ['url'],
textValues: ['', 'env:DATABASE_URL'],
multiselectValues: [['orbit_analytics', 'orbit_raw']],
});
const testConnection = vi.fn(async () => 0);
const scanConnection = vi.fn(async asyncScanProjectDir => {
@ -1011,6 +1456,19 @@ describe('setup databases step', () => {
return 0;
});
const listSchemas = vi.fn(async () => ['orbit_analytics', 'orbit_raw', 'public']);
const listTables = vi.fn(async () => [
{ schema: 'orbit_analytics', name: 'events', kind: 'table' as const },
{ schema: 'orbit_raw', name: 'inputs', kind: 'table' as const },
{ schema: 'public', name: 'misc', kind: 'table' as const },
]);
const pickers = makePickerStubs({
scopes: [
{
schemas: ['orbit_analytics', 'orbit_raw'],
tables: ['orbit_analytics.events', 'orbit_raw.inputs'],
},
],
});
const result = await runKtxSetupDatabasesStep(
{
@ -1021,20 +1479,24 @@ describe('setup databases step', () => {
skipDatabases: false,
},
io.io,
{ prompts, testConnection, scanConnection, listSchemas },
{
prompts,
testConnection,
scanConnection,
listSchemas,
listTables,
pickDatabaseScope: pickers.pickDatabaseScope,
},
);
expect(result.status).toBe('ready');
expect(listSchemas).toHaveBeenCalledWith(tempDir, 'postgres-warehouse');
expect(prompts.multiselect).toHaveBeenCalledWith({
message: expect.stringContaining('PostgreSQL schemas to scan'),
options: [
{ value: 'orbit_analytics', label: 'orbit_analytics' },
{ value: 'orbit_raw', label: 'orbit_raw' },
{ value: 'public', label: 'public' },
],
initialValues: ['orbit_analytics', 'orbit_raw'],
required: true,
expect(pickers.scopeCalls).toHaveLength(1);
expect(pickers.scopeCalls[0]).toMatchObject({
connectionId: 'postgres-warehouse',
schemaNoun: 'schema',
schemaNounPlural: 'schemas',
defaultSchemas: ['orbit_analytics', 'orbit_raw'],
});
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
expect(config.connections['postgres-warehouse']).toMatchObject({

738
packages/cli/src/setup-databases.ts
View file

File diff suppressed because it is too large Load diff

14
packages/cli/src/setup-embeddings.test.ts
View file

@ -319,14 +319,14 @@ describe('setup embeddings step', () => {
projectDir: tempDir,
inputMode: 'disabled',
embeddingBackend: 'openai',
embeddingApiKeyEnv: 'OPENAI_API_KEY',
embeddingApiKeyEnv: 'OPENAI_API_KEY', // pragma: allowlist secret
cliVersion: '0.2.0',
runtimeInstallPolicy: 'auto',
skipEmbeddings: false,
},
io.io,
{
env: { OPENAI_API_KEY: 'sk-openai-test' },
env: { OPENAI_API_KEY: 'sk-openai-test' }, // pragma: allowlist secret
healthCheck,
},
);
@ -336,14 +336,14 @@ describe('setup embeddings step', () => {
backend: 'openai',
model: 'text-embedding-3-small',
dimensions: 1536,
openai: { apiKey: 'sk-openai-test' },
openai: { apiKey: 'sk-openai-test' }, // pragma: allowlist secret
});
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
expect(config.ingest.embeddings).toMatchObject({
backend: 'openai',
model: 'text-embedding-3-small',
dimensions: 1536,
openai: { api_key: 'env:OPENAI_API_KEY' },
openai: { api_key: 'env:OPENAI_API_KEY' }, // pragma: allowlist secret
});
expect(io.stdout()).not.toContain('sk-openai-test');
});
@ -367,7 +367,7 @@ describe('setup embeddings step', () => {
io.io,
{
prompts,
env: { OPENAI_API_KEY: 'sk-openai-test' },
env: { OPENAI_API_KEY: 'sk-openai-test' }, // pragma: allowlist secret
healthCheck,
ensureLocalEmbeddings: vi.fn(async () => managedDaemon()),
},
@ -384,7 +384,7 @@ describe('setup embeddings step', () => {
backend: 'openai',
model: 'text-embedding-3-small',
dimensions: 1536,
openai: { apiKey: 'sk-openai-test' },
openai: { apiKey: 'sk-openai-test' }, // pragma: allowlist secret
});
expect(prompts.select).toHaveBeenCalledWith(
expect.objectContaining({
@ -478,7 +478,7 @@ describe('setup embeddings step', () => {
},
makeIo().io,
{
env: { OPENAI_API_KEY: 'sk-openai-test' },
env: { OPENAI_API_KEY: 'sk-openai-test' }, // pragma: allowlist secret
healthCheck,
},
),

87
packages/cli/src/setup-models.test.ts
View file

@ -7,10 +7,8 @@ import {
BUNDLED_ANTHROPIC_MODELS,
fetchAnthropicModels,
type KtxSetupModelPromptAdapter,
runKtxSetupGcloudApplicationDefaultAuth,
runKtxSetupAnthropicModelStep,
} from './setup-models.js';
import type { KtxCliIo } from './cli-runtime.js';
function makeIo() {
let stdout = '';
@ -34,6 +32,17 @@ function makeIo() {
};
}
function makeSpinnerEvents() {
const events: string[] = [];
const spinner = vi.fn(() => ({
start: (msg: string) => events.push(`start:${msg}`),
message: (msg: string) => events.push(`message:${msg}`),
stop: (msg: string) => events.push(`stop:${msg}`),
error: (msg: string) => events.push(`error:${msg}`),
}));
return { events, spinner };
}
function makePromptAdapter(options: {
providerChoice?: string;
selectValues?: string[];
@ -191,6 +200,7 @@ describe('setup Anthropic model step', () => {
it('configures env credentials, selected model, prompt caching, and llm completion state', async () => {
const io = makeIo();
const { events: spinnerEvents, spinner } = makeSpinnerEvents();
const result = await runKtxSetupAnthropicModelStep(
{
projectDir: tempDir,
@ -203,6 +213,7 @@ describe('setup Anthropic model step', () => {
{
env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, // pragma: allowlist secret
healthCheck: vi.fn(async () => ({ ok: true as const })),
spinner,
},
);
@ -219,6 +230,10 @@ describe('setup Anthropic model step', () => {
expect(config.scan.enrichment.mode).toBe('llm');
expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:');
expect((await readKtxSetupState(tempDir)).completed_steps).toContain('llm');
expect(spinnerEvents).toEqual([
'start:Checking Anthropic API LLM (claude-sonnet-4-6).',
'stop:LLM test passed (Anthropic API, claude-sonnet-4-6)',
]);
expect(io.stdout()).toContain('LLM ready: yes');
expect(io.stdout()).not.toContain('sk-ant-test');
});
@ -226,6 +241,7 @@ describe('setup Anthropic model step', () => {
it('configures Vertex AI provider, selected model, prompt caching, and llm completion state', async () => {
const io = makeIo();
const healthCheck = vi.fn(async () => ({ ok: true as const }));
const { events: spinnerEvents, spinner } = makeSpinnerEvents();
const result = await runKtxSetupAnthropicModelStep(
{
@ -238,7 +254,7 @@ describe('setup Anthropic model step', () => {
skipLlm: false,
},
io.io,
{ env: {}, healthCheck },
{ env: {}, healthCheck, spinner },
);
expect(result.status).toBe('ready');
@ -260,13 +276,16 @@ describe('setup Anthropic model step', () => {
expect(config.scan.enrichment.mode).toBe('llm');
expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:');
expect((await readKtxSetupState(tempDir)).completed_steps).toContain('llm');
expect(spinnerEvents).toEqual([
'start:Checking Vertex AI LLM (claude-sonnet-4-6).',
'stop:LLM test passed (Vertex AI, claude-sonnet-4-6)',
]);
expect(io.stdout()).toContain('LLM ready: yes (claude-sonnet-4-6)');
});
it('can run gcloud auth for Vertex AI and infer project and default location', async () => {
it('uses existing Vertex AI credentials without offering to run gcloud auth', async () => {
const io = makeIo();
const prompts = makePromptAdapter({ selectValues: ['vertex', 'gcloud', 'local-gcp-project', 'claude-sonnet-4-6'] });
const runGcloudAuth = vi.fn(async () => ({ ok: true as const }));
const prompts = makePromptAdapter({ selectValues: ['vertex', 'existing', 'local-gcp-project', 'claude-sonnet-4-6'] });
const readGcloudProject = vi.fn(async () => 'local-gcp-project');
const listGcloudProjects = vi.fn(async () => [
{ projectId: 'local-gcp-project', name: 'Local project' },
@ -280,7 +299,6 @@ describe('setup Anthropic model step', () => {
{
prompts,
env: {},
runGcloudAuth,
readGcloudProject,
listGcloudProjects,
healthCheck,
@ -288,7 +306,15 @@ describe('setup Anthropic model step', () => {
);
expect(result.status).toBe('ready');
expect(runGcloudAuth).toHaveBeenCalledWith(io.io);
expect(prompts.select).toHaveBeenCalledWith(
expect.objectContaining({
message: expect.stringContaining('How should KTX authenticate with Google Vertex AI?'),
options: [
{ value: 'existing', label: 'Use existing gcloud/Application Default Credentials' },
{ value: 'back', label: 'Back' },
],
}),
);
expect(readGcloudProject).toHaveBeenCalled();
expect(listGcloudProjects).toHaveBeenCalled();
expect(prompts.text).not.toHaveBeenCalled();
@ -303,6 +329,22 @@ describe('setup Anthropic model step', () => {
],
}),
);
expect(prompts.select).toHaveBeenCalledWith(
expect.objectContaining({
message: expect.stringContaining('Which Anthropic model should KTX use?'),
options: [
{ value: 'claude-opus-4-7', label: 'Claude Opus 4.7' },
{ value: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6' },
{ value: 'claude-opus-4-6', label: 'Claude Opus 4.6' },
{ value: 'claude-opus-4-5', label: 'Claude Opus 4.5' },
{ value: 'claude-haiku-4-5', label: 'Claude Haiku 4.5' },
{ value: 'claude-sonnet-4-5', label: 'Claude Sonnet 4.5' },
{ value: 'claude-opus-4-1', label: 'Claude Opus 4.1' },
{ value: 'manual', label: 'Enter a model ID manually' },
{ value: 'back', label: 'Back' },
],
}),
);
expect(healthCheck).toHaveBeenCalledWith({
backend: 'vertex',
vertex: { project: 'local-gcp-project', location: 'us-east5' },
@ -415,35 +457,6 @@ describe('setup Anthropic model step', () => {
);
});
it('runs only gcloud application-default login for Vertex AI auth', async () => {
const io = makeIo();
const runGcloud = vi.fn(async () => ({ ok: true as const }));
await expect(runKtxSetupGcloudApplicationDefaultAuth(io.io, runGcloud)).resolves.toEqual({ ok: true });
expect(runGcloud).toHaveBeenCalledTimes(1);
expect(runGcloud).toHaveBeenCalledWith(['auth', 'application-default', 'login'], expect.anything());
expect(runGcloud).not.toHaveBeenCalledWith(['auth', 'login'], expect.anything());
expect(io.stdout()).toContain('gcloud auth application-default login');
expect(io.stdout()).not.toContain('gcloud auth login');
});
it('indents gcloud auth output inside the setup gutter', async () => {
const io = makeIo();
const runGcloud = vi.fn(async (_args: string[], commandIo: KtxCliIo) => {
commandIo.stdout.write('Your browser has been opened to visit:\n\n https://accounts.example/auth\n');
commandIo.stderr.write('Credentials saved to file: [/tmp/application_default_credentials.json]\n');
return { ok: true as const };
});
await expect(runKtxSetupGcloudApplicationDefaultAuth(io.io, runGcloud)).resolves.toEqual({ ok: true });
expect(io.stdout()).toContain('│ Your browser has been opened to visit:');
expect(io.stdout()).toContain('│ https://accounts.example/auth');
expect(io.stderr()).toContain('│ Credentials saved to file: [/tmp/application_default_credentials.json]');
expect(io.stdout()).not.toContain('\nYour browser has been opened');
});
it('explains common Vertex AI Forbidden health-check causes', async () => {
const io = makeIo();

176
packages/cli/src/setup-models.ts
View file

@ -1,4 +1,4 @@
import { execFile, spawn } from 'node:child_process';
import { execFile } from 'node:child_process';
import { writeFile } from 'node:fs/promises';
import { promisify } from 'node:util';
import { resolveLocalKtxLlmConfig } from '@ktx/context';
@ -11,6 +11,7 @@ import {
serializeKtxProjectConfig,
} from '@ktx/context/project';
import { type KtxLlmConfig, type KtxLlmHealthCheckResult, runKtxLlmHealthCheck } from '@ktx/llm';
import { createClackSpinner, type KtxCliSpinner } from './clack.js';
import type { KtxCliIo } from './cli-runtime.js';
import { withTextInputNavigation } from './prompt-navigation.js';
import { envCredentialReference, writeProjectLocalSecretReference } from './setup-secrets.js';
@ -61,9 +62,9 @@ export interface KtxSetupModelDeps {
prompts?: KtxSetupModelPromptAdapter;
listModels?: (apiKey: string) => Promise<AnthropicModelChoice[]>;
healthCheck?: (config: KtxLlmConfig) => Promise<KtxLlmHealthCheckResult>;
runGcloudAuth?: (io: KtxCliIo) => Promise<GcloudAuthResult>;
readGcloudProject?: () => Promise<string | undefined>;
listGcloudProjects?: () => Promise<GcloudProjectChoice[]>;
spinner?: () => KtxCliSpinner;
}
export const BUNDLED_ANTHROPIC_MODEL_REGISTRY_VERSION = '2026-05-07';
@ -74,6 +75,16 @@ export const BUNDLED_ANTHROPIC_MODELS: AnthropicModelChoice[] = [
{ id: 'claude-haiku-4-5', label: 'Claude Haiku 4.5', recommended: false },
];
const VERTEX_ANTHROPIC_MODELS: AnthropicModelChoice[] = [
{ id: 'claude-opus-4-7', label: 'Claude Opus 4.7', recommended: false },
{ id: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6', recommended: false },
{ id: 'claude-opus-4-6', label: 'Claude Opus 4.6', recommended: false },
{ id: 'claude-opus-4-5', label: 'Claude Opus 4.5', recommended: false },
{ id: 'claude-haiku-4-5', label: 'Claude Haiku 4.5', recommended: false },
{ id: 'claude-sonnet-4-5', label: 'Claude Sonnet 4.5', recommended: false },
{ id: 'claude-opus-4-1', label: 'Claude Opus 4.1', recommended: false },
];
const HIDDEN_ANTHROPIC_MODEL_PATTERNS = [
/^claude-sonnet-4$/i,
/^claude-opus-4$/i,
@ -91,8 +102,8 @@ const ANTHROPIC_MODEL_PROMPT_CONTEXT =
'into semantic-layer sources and wiki context.';
const VERTEX_AUTH_PROMPT_CONTEXT =
'KTX can use Google Cloud Application Default Credentials for local Vertex AI access. This opens the normal ' +
'gcloud browser login flow and does not store Google credentials in ktx.yaml.';
'KTX uses Google Cloud Application Default Credentials for local Vertex AI access and does not store Google ' +
'credentials in ktx.yaml. If needed, run gcloud auth application-default login before continuing.';
const VERTEX_PROJECT_PROMPT_CONTEXT =
'KTX stores the selected Google Cloud project ID in ktx.yaml and uses Application Default Credentials for ' +
'access. Project visibility depends on the signed-in Google account and organization permissions.';
@ -137,94 +148,17 @@ type VertexConfigChoice =
}
| { status: 'back' | 'missing-input' };
type VertexAuthChoice = { status: 'ready' } | { status: 'back' | 'missing-input' };
type VertexAuthChoice = { status: 'ready' } | { status: 'back' };
export type GcloudAuthResult = { ok: true } | { ok: false; message: string };
interface GcloudProjectChoice {
projectId: string;
name?: string;
}
type GcloudCommandRunner = (args: string[], io: KtxCliIo) => Promise<GcloudAuthResult>;
function createPromptAdapter(): KtxSetupModelPromptAdapter {
return createKtxSetupPromptAdapter({ selectCancelValue: 'back' });
}
function createIndentedCommandIo(io: KtxCliIo): KtxCliIo {
const indentedWriter = (write: (chunk: string) => void) => {
let atLineStart = true;
return (chunk: string) => {
for (const char of chunk) {
if (atLineStart) {
write('│ ');
atLineStart = false;
}
write(char);
if (char === '\n') {
atLineStart = true;
}
}
};
};
return {
stdout: {
isTTY: io.stdout.isTTY,
columns: io.stdout.columns,
write: indentedWriter((chunk) => io.stdout.write(chunk)),
},
stderr: {
write: indentedWriter((chunk) => io.stderr.write(chunk)),
},
};
}
function runInteractiveGcloud(args: string[], io: KtxCliIo): Promise<GcloudAuthResult> {
return new Promise((resolve) => {
let settled = false;
const child = spawn('gcloud', args, { stdio: ['inherit', 'pipe', 'pipe'] });
child.stdout?.on('data', (chunk: Buffer) => {
io.stdout.write(chunk.toString('utf8'));
});
child.stderr?.on('data', (chunk: Buffer) => {
io.stderr.write(chunk.toString('utf8'));
});
child.on('error', (error: NodeJS.ErrnoException) => {
if (settled) {
return;
}
settled = true;
if (error.code === 'ENOENT') {
resolve({ ok: false, message: 'gcloud CLI was not found on PATH.' });
return;
}
resolve({ ok: false, message: error.message });
});
child.on('close', (code, signal) => {
if (settled) {
return;
}
settled = true;
if (code === 0) {
resolve({ ok: true });
return;
}
resolve({
ok: false,
message: signal ? `gcloud exited after signal ${signal}.` : `gcloud exited with code ${code ?? 'unknown'}.`,
});
});
});
}
export async function runKtxSetupGcloudApplicationDefaultAuth(
io: KtxCliIo,
runGcloud: GcloudCommandRunner = runInteractiveGcloud,
): Promise<GcloudAuthResult> {
io.stdout.write('│ Running gcloud auth application-default login...\n');
return await runGcloud(['auth', 'application-default', 'login'], createIndentedCommandIo(io));
}
async function defaultReadGcloudProject(): Promise<string | undefined> {
try {
const { stdout } = await execFileAsync('gcloud', ['config', 'get-value', 'project'], { encoding: 'utf8' });
@ -374,6 +308,53 @@ function buildVertexHealthConfig(vertex: { project?: string; location: string },
};
}
type LlmHealthProvider = 'Anthropic API' | 'Vertex AI';
function llmHealthCheckStartText(provider: LlmHealthProvider, model: string): string {
return `Checking ${provider} LLM (${model}).`;
}
function startLlmHealthCheckProgress(
spinner: KtxCliSpinner,
message: string,
): { succeed(msg: string): void; fail(msg: string): void } {
spinner.start(message);
return {
succeed(msg: string) {
spinner.stop(msg);
},
fail(msg: string) {
spinner.error(msg);
},
};
}
async function runLlmHealthCheckWithProgress(
config: KtxLlmConfig,
provider: LlmHealthProvider,
model: string,
healthCheck: (config: KtxLlmConfig) => Promise<KtxLlmHealthCheckResult>,
deps: KtxSetupModelDeps,
): Promise<KtxLlmHealthCheckResult> {
const progress = startLlmHealthCheckProgress(
(deps.spinner ?? createClackSpinner)(),
llmHealthCheckStartText(provider, model),
);
let health: KtxLlmHealthCheckResult;
try {
health = await healthCheck(config);
} catch (error) {
progress.fail('LLM test failed');
throw error;
}
if (health.ok) {
progress.succeed(`LLM test passed (${provider}, ${model})`);
} else {
progress.fail('LLM test failed');
}
return health;
}
function formatVertexHealthFailure(message: string, vertex: { project?: string; location: string }): string {
const trimmed = message.trim() || 'unknown error';
if (!/(forbidden|permission|permission_denied|403)/i.test(trimmed)) {
@ -516,7 +497,6 @@ async function chooseBackend(
async function chooseVertexAuth(
args: KtxSetupModelArgs,
io: KtxCliIo,
deps: KtxSetupModelDeps,
): Promise<VertexAuthChoice> {
if (args.inputMode === 'disabled' || args.vertexProject || args.vertexLocation) {
@ -527,7 +507,6 @@ async function chooseVertexAuth(
const choice = await prompts.select({
message: `How should KTX authenticate with Google Vertex AI?\n\n${VERTEX_AUTH_PROMPT_CONTEXT}`,
options: [
{ value: 'gcloud', label: 'Run gcloud Application Default Credentials login' },
{ value: 'existing', label: 'Use existing gcloud/Application Default Credentials' },
{ value: 'back', label: 'Back' },
],
@ -535,15 +514,6 @@ async function chooseVertexAuth(
if (choice === 'back') {
return { status: 'back' };
}
if (choice !== 'gcloud') {
return { status: 'ready' };
}
const result = await (deps.runGcloudAuth ?? runKtxSetupGcloudApplicationDefaultAuth)(io);
if (!result.ok) {
io.stderr.write(`gcloud authentication failed: ${result.message}\n`);
return { status: 'missing-input' };
}
return { status: 'ready' };
}
@ -799,7 +769,7 @@ async function chooseVertexModel(args: KtxSetupModelArgs, io: KtxCliIo, deps: Kt
return { status: 'missing-input' };
}
const selectableModels = BUNDLED_ANTHROPIC_MODELS.filter(isSelectableAnthropicModel);
const selectableModels = VERTEX_ANTHROPIC_MODELS.filter(isSelectableAnthropicModel);
const prompts = deps.prompts ?? createPromptAdapter();
const choice = await prompts.select({
message: `Which Anthropic model should KTX use?\n\n${ANTHROPIC_MODEL_PROMPT_CONTEXT}`,
@ -901,7 +871,7 @@ export async function runKtxSetupAnthropicModelStep(
: attemptArgs;
if (backendChoice.backend === 'vertex') {
const auth = await chooseVertexAuth(backendArgs, io, deps);
const auth = await chooseVertexAuth(backendArgs, deps);
if (auth.status === 'back' && backendChoice.prompted) {
attemptArgs = buildInteractiveRetryArgs(args);
continue;
@ -931,7 +901,13 @@ export async function runKtxSetupAnthropicModelStep(
return { status: model.status, projectDir: args.projectDir };
}
const health = await healthCheck(buildVertexHealthConfig(vertex.values, model.model));
const health = await runLlmHealthCheckWithProgress(
buildVertexHealthConfig(vertex.values, model.model),
'Vertex AI',
model.model,
healthCheck,
deps,
);
if (health.ok) {
await persistLlmConfig(args.projectDir, { backend: 'vertex', vertex: vertex.refs }, model.model);
io.stdout.write(`│ LLM ready: yes (${model.model})\n`);
@ -973,7 +949,13 @@ export async function runKtxSetupAnthropicModelStep(
return { status: model.status, projectDir: args.projectDir };
}
const health = await healthCheck(buildAnthropicHealthConfig(credential.value, model.model));
const health = await runLlmHealthCheckWithProgress(
buildAnthropicHealthConfig(credential.value, model.model),
'Anthropic API',
model.model,
healthCheck,
deps,
);
if (health.ok) {
await persistLlmConfig(args.projectDir, { backend: 'anthropic', credentialRef: credential.ref }, model.model);
io.stdout.write(`│ LLM ready: yes (${model.model})\n`);

313
packages/cli/src/setup-sources.test.ts
View file

@ -862,6 +862,7 @@ describe('setup sources step', () => {
message: 'Configure dbt',
options: [
{ value: 'existing:dbt-main', label: 'Use existing dbt connection: dbt-main' },
{ value: 'edit:dbt-main', label: 'Edit existing dbt connection: dbt-main' },
{ value: 'new', label: 'Add new dbt connection' },
{ value: 'back', label: 'Back' },
],
@ -989,6 +990,10 @@ describe('setup sources step', () => {
value: `existing:${testCase.connectionId}`,
label: `Use existing ${testCase.expectedLabel} connection: ${testCase.connectionId}`,
},
{
value: `edit:${testCase.connectionId}`,
label: `Edit existing ${testCase.expectedLabel} connection: ${testCase.connectionId}`,
},
{ value: 'new', label: `Add new ${testCase.expectedLabel} connection` },
{ value: 'back', label: 'Back' },
],
@ -997,6 +1002,314 @@ describe('setup sources step', () => {
}
});
it('edits an existing Notion source and reopens the page picker with stored pages selected', async () => {
await addPrimarySource();
await addConnection('notion-main', {
driver: 'notion',
auth_token_ref: 'env:NOTION_TOKEN',
crawl_mode: 'selected_roots',
root_page_ids: ['old-page'],
root_database_ids: [],
root_data_source_ids: [],
});
const validateNotion = vi.fn(async () => ({ ok: true as const, detail: 'roots=1' }));
const pickNotionRootPages = vi.fn(async () => ({ kind: 'selected' as const, rootPageIds: ['new-page'] }));
const testPrompts = prompts({
multiselect: [['notion']],
select: ['edit:notion-main', 'keep', 'selected_roots', 'done'],
});
await expect(
runKtxSetupSourcesStep(
{ projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false },
makeIo().io,
{
prompts: testPrompts,
validateNotion,
pickNotionRootPages,
},
),
).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['notion-main'] });
expect(testPrompts.select).toHaveBeenCalledWith({
message: 'How should KTX find your Notion integration token?',
options: [
{ value: 'keep', label: 'Keep existing credential' },
{ value: 'env', label: 'Use NOTION_TOKEN from the environment' },
{ value: 'paste', label: 'Paste a key and save it as a local secret file' },
{ value: 'back', label: 'Back' },
],
});
expect(pickNotionRootPages).toHaveBeenCalledWith(
{
connectionId: 'notion-main',
connection: expect.objectContaining({
driver: 'notion',
auth_token_ref: 'env:NOTION_TOKEN',
crawl_mode: 'selected_roots',
root_page_ids: ['old-page'],
}),
},
expect.anything(),
);
expect((await readConfig()).connections['notion-main']).toMatchObject({
driver: 'notion',
auth_token_ref: 'env:NOTION_TOKEN',
crawl_mode: 'selected_roots',
root_page_ids: ['new-page'],
});
});
it('edits an existing Metabase source with the current URL and credential as defaults', async () => {
await addPrimarySource();
await addConnection('metabase-main', {
driver: 'metabase',
api_url: 'https://metabase-old.example.com',
api_key_ref: 'env:METABASE_API_KEY', // pragma: allowlist secret
mappings: {
databaseMappings: { '1': 'warehouse' },
syncEnabled: { '1': true },
syncMode: 'ALL',
},
});
const testPrompts = prompts({
multiselect: [['metabase']],
select: ['edit:metabase-main', 'keep', 'done'],
text: ['https://metabase-new.example.com'],
});
const discoverMetabaseDatabases = vi.fn(async () => [
{ id: 2, name: 'Analytics', engine: 'postgres', host: 'db.example.com', dbName: 'analytics' },
]);
await expect(
runKtxSetupSourcesStep(
{ projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false },
makeIo().io,
{
prompts: testPrompts,
discoverMetabaseDatabases,
validateMetabase: vi.fn(async () => ({ ok: true as const, detail: 'mapping validated' })),
runMapping: vi.fn(async () => 0),
},
),
).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['metabase-main'] });
expect(testPrompts.text).toHaveBeenCalledWith({
message: textInputPrompt('Metabase URL'),
initialValue: 'https://metabase-old.example.com',
});
expect(testPrompts.select).toHaveBeenCalledWith({
message: 'How should KTX find your Metabase API key?',
options: [
{ value: 'keep', label: 'Keep existing credential' },
{ value: 'env', label: 'Use METABASE_API_KEY from the environment' },
{ value: 'paste', label: 'Paste a key and save it as a local secret file' },
{ value: 'back', label: 'Back' },
],
});
expect(discoverMetabaseDatabases).toHaveBeenCalledWith({
sourceUrl: 'https://metabase-new.example.com',
sourceApiKeyRef: 'env:METABASE_API_KEY',
sourceConnectionId: 'metabase-main',
});
expect((await readConfig()).connections['metabase-main']).toMatchObject({
driver: 'metabase',
api_url: 'https://metabase-new.example.com',
api_key_ref: 'env:METABASE_API_KEY',
mappings: {
databaseMappings: { '2': 'warehouse' },
syncEnabled: { '2': true },
syncMode: 'ALL',
},
});
});
it('rolls back an edited context source when validation fails', async () => {
await addPrimarySource();
await addConnection('dbt-main', {
driver: 'dbt',
source_dir: '/repo/existing-dbt',
project_name: 'analytics',
});
const validateDbt = vi.fn(async () => ({ ok: false as const, message: 'dbt project not found' }));
const testPrompts = prompts({
multiselect: [['dbt']],
select: ['edit:dbt-main', 'path'],
text: ['/repo/new-dbt', ''],
});
await expect(
runKtxSetupSourcesStep(
{ projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false },
makeIo().io,
{
prompts: testPrompts,
validateDbt,
},
),
).resolves.toEqual({ status: 'failed', projectDir });
expect(validateDbt).toHaveBeenCalledWith(expect.objectContaining({
driver: 'dbt',
source_dir: '/repo/new-dbt',
}));
const config = await readConfig();
expect(config.connections['dbt-main']).toMatchObject({
driver: 'dbt',
source_dir: '/repo/existing-dbt',
project_name: 'analytics',
});
expect(config.ingest.adapters).not.toContain('dbt');
});
it('lets git-backed context source edits keep the existing repo credential', async () => {
await addPrimarySource();
await addConnection('metricflow-main', {
driver: 'metricflow',
metricflow: {
repoUrl: 'https://github.com/acme/private-metricflow',
branch: 'main',
path: 'metrics',
auth_token_ref: 'env:METRICFLOW_REPO_TOKEN', // pragma: allowlist secret
},
});
const testGitRepo = vi.fn(async () => ({ ok: false as const, error: 'authentication required' }));
const testPrompts = prompts({
multiselect: [['metricflow']],
select: ['edit:metricflow-main', 'git', 'keep', 'done'],
text: ['https://github.com/acme/private-metricflow', 'main', 'metrics'],
});
await expect(
runKtxSetupSourcesStep(
{ projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false },
makeIo().io,
{
prompts: testPrompts,
testGitRepo,
validateMetricflow: vi.fn(async () => ({ ok: true as const, detail: 'metrics=1' })),
},
),
).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['metricflow-main'] });
expect(testPrompts.select).toHaveBeenCalledWith({
message: 'This MetricFlow repo requires authentication.',
options: [
{ value: 'keep', label: 'Keep existing credential' },
{ value: 'env', label: 'Use GITHUB_TOKEN from the environment' },
{ value: 'paste', label: 'Paste a token and save it as a local secret file' },
{ value: 'skip', label: 'Skip — try without authentication' },
{ value: 'back', label: 'Back' },
],
});
expect((await readConfig()).connections['metricflow-main']).toMatchObject({
driver: 'metricflow',
metricflow: {
repoUrl: 'https://github.com/acme/private-metricflow',
branch: 'main',
path: 'metrics',
auth_token_ref: 'env:METRICFLOW_REPO_TOKEN',
},
});
});
it('edits an existing context source from the configured-source follow-up menu', async () => {
await addPrimarySource();
await addConnection('dbt-main', {
driver: 'dbt',
source_dir: '/repo/existing-dbt',
project_name: 'analytics',
});
const validateDbt = vi.fn(async () => ({ ok: true as const, detail: 'project=analytics schemas=2' }));
const testPrompts = prompts({
multiselect: [['dbt']],
select: ['existing:dbt-main', 'edit', 'dbt-main', 'path', 'done'],
text: ['/repo/edited-dbt', ''],
});
await expect(
runKtxSetupSourcesStep(
{ projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false },
makeIo().io,
{
prompts: testPrompts,
validateDbt,
},
),
).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['dbt-main'] });
expect(testPrompts.select).toHaveBeenCalledWith({
message: '1 context source configured (dbt-main). Add another?',
options: [
{ value: 'done', label: 'Done — continue to context build' },
{ value: 'edit', label: 'Edit an existing context source' },
{ value: 'add', label: 'Add another context source' },
],
});
expect(testPrompts.select).toHaveBeenCalledWith({
message: 'Context source to edit',
options: [
{ value: 'dbt-main', label: 'dbt-main (dbt)' },
{ value: 'back', label: 'Back' },
],
});
expect(testPrompts.text).toHaveBeenCalledWith({
message: textInputPrompt('dbt local path'),
initialValue: '/repo/existing-dbt',
});
expect(validateDbt).toHaveBeenLastCalledWith(expect.objectContaining({
driver: 'dbt',
source_dir: '/repo/edited-dbt',
}));
expect((await readConfig()).connections['dbt-main']).toMatchObject({
driver: 'dbt',
source_dir: '/repo/edited-dbt',
project_name: 'analytics',
});
});
it('backs out of editing an existing context source to the source connection menu', async () => {
await addPrimarySource();
await addConnection('dbt-main', {
driver: 'dbt',
source_dir: '/repo/existing-dbt',
project_name: 'analytics',
});
const validateDbt = vi.fn(async () => ({ ok: true as const, detail: 'project=analytics schemas=2' }));
const testPrompts = prompts({
multiselect: [['dbt']],
select: ['edit:dbt-main', 'back', 'existing:dbt-main'],
});
await expect(
runKtxSetupSourcesStep(
{ projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false },
makeIo().io,
{
prompts: testPrompts,
validateDbt,
},
),
).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['dbt-main'] });
expect(
vi
.mocked(testPrompts.select)
.mock.calls.map(([options]) => options.message)
.filter((message) => message === 'Configure dbt'),
).toHaveLength(2);
expect(validateDbt).toHaveBeenCalledWith({
driver: 'dbt',
source_dir: '/repo/existing-dbt',
project_name: 'analytics',
});
expect((await readConfig()).connections['dbt-main']).toMatchObject({
driver: 'dbt',
source_dir: '/repo/existing-dbt',
project_name: 'analytics',
});
});
it('lets Escape from dbt git URL return to source location selection', async () => {
await addPrimarySource();
const validateDbt = vi.fn(async () => ({ ok: true as const, detail: 'project=analytics schemas=2' }));

488
packages/cli/src/setup-sources.ts
View file

@ -224,17 +224,20 @@ async function chooseSourceCredentialRef(input: {
label: string;
envName: string;
secretFileName: string;
existingRef?: string;
}): Promise<string | 'back'> {
while (true) {
const choice = await input.prompts.select({
message: `How should KTX find your ${input.label}?`,
options: [
...(input.existingRef ? [{ value: 'keep', label: 'Keep existing credential' }] : []),
{ value: 'env', label: `Use ${input.envName} from the environment` },
{ value: 'paste', label: 'Paste a key and save it as a local secret file' },
{ value: 'back', label: 'Back' },
],
});
if (choice === 'back') return 'back';
if (choice === 'keep' && input.existingRef) return input.existingRef;
if (choice === 'paste') {
const value = await input.prompts.password({ message: input.label });
if (value === undefined) continue;
@ -256,12 +259,14 @@ async function chooseGitAuthCredentialRef(input: {
projectDir: string;
source: KtxSetupSourceType;
connectionId: string;
existingRef?: string;
}): Promise<string | undefined | 'back'> {
const label = input.source === 'dbt' ? 'This' : `This ${sourceLabel(input.source)}`;
while (true) {
const choice = await input.prompts.select({
message: `${label} repo requires authentication.`,
options: [
...(input.existingRef ? [{ value: 'keep', label: 'Keep existing credential' }] : []),
{ value: 'env', label: 'Use GITHUB_TOKEN from the environment' },
{ value: 'paste', label: 'Paste a token and save it as a local secret file' },
{ value: 'skip', label: 'Skip — try without authentication' },
@ -269,6 +274,7 @@ async function chooseGitAuthCredentialRef(input: {
],
});
if (choice === 'back') return 'back';
if (choice === 'keep' && input.existingRef) return input.existingRef;
if (choice === 'skip') return undefined;
if (choice === 'paste') {
const value = await input.prompts.password({ message: 'Git access token' });
@ -792,8 +798,14 @@ interface WarehouseConnectionChoice {
type InteractiveSourceConnectionChoice =
| { kind: 'existing'; connectionId: string; connection: KtxProjectConnectionConfig }
| { kind: 'new'; args: KtxSetupSourcesArgs }
| { kind: 'edited'; connectionId: string; args: KtxSetupSourcesArgs }
| 'back';
type SourceSetupChoiceResult =
| { status: 'ready'; connectionId: string }
| { status: 'back' }
| { status: 'failed' };
async function runSourcePromptSteps(
initialState: SourcePromptState,
stepsForState: (state: SourcePromptState) => SourcePromptStep[],
@ -827,6 +839,12 @@ function resetRepoLocationFields(state: SourcePromptState): void {
delete state.sourceProjectName;
}
function sourceLocationFromArgs(args: KtxSetupSourcesArgs): SourceLocationChoice | undefined {
if (args.sourcePath) return 'path';
if (args.sourceGitUrl) return 'git';
return undefined;
}
function warehouseConnectionChoices(config: KtxProjectConfig): WarehouseConnectionChoice[] {
return Object.entries(config.connections)
.filter(([, connection]) => PRIMARY_SOURCE_DRIVERS.has(String(connection.driver ?? '').toLowerCase()))
@ -963,7 +981,7 @@ async function promptForInteractiveSource(
testGitRepo: KtxSetupSourcesDeps['testGitRepo'] = testRepoConnection,
discoverMetabaseDatabaseList?: KtxSetupSourcesDeps['discoverMetabaseDatabases'],
): Promise<KtxSetupSourcesArgs | 'back'> {
const initialState: SourcePromptState = { ...args, source };
const initialState: SourcePromptState = { ...args, source, sourceLocation: sourceLocationFromArgs(args) };
if (args.sourceConnectionId) {
initialState.sourceConnectionId = args.sourceConnectionId;
}
@ -993,7 +1011,10 @@ async function promptForInteractiveSource(
...(state.sourceLocation === 'path'
? [
async (currentState: SourcePromptState) => {
const sourcePath = await promptText(prompts, { message: `${source} local path` });
const sourcePath = await promptText(prompts, {
message: `${source} local path`,
...(currentState.sourcePath ? { initialValue: currentState.sourcePath } : {}),
});
if (sourcePath === undefined) return 'back';
currentState.sourcePath = sourcePath;
return 'next';
@ -1003,13 +1024,19 @@ async function promptForInteractiveSource(
...(state.sourceLocation === 'git'
? [
async (currentState: SourcePromptState) => {
const sourceGitUrl = await promptText(prompts, { message: `${source} git URL` });
const sourceGitUrl = await promptText(prompts, {
message: `${source} git URL`,
...(currentState.sourceGitUrl ? { initialValue: currentState.sourceGitUrl } : {}),
});
if (sourceGitUrl === undefined) return 'back';
currentState.sourceGitUrl = sourceGitUrl;
return 'next';
},
async (currentState: SourcePromptState) => {
const branch = await promptText(prompts, { message: `${source} git branch`, initialValue: 'main' });
const branch = await promptText(prompts, {
message: `${source} git branch`,
initialValue: currentState.sourceBranch ?? 'main',
});
if (branch === undefined) return 'back';
currentState.sourceBranch = branch || 'main';
return 'next';
@ -1030,6 +1057,7 @@ async function promptForInteractiveSource(
projectDir: args.projectDir,
source,
connectionId: currentState.sourceConnectionId ?? `${source}-main`,
existingRef: currentState.sourceAuthTokenRef,
});
if (authRef === 'back') return 'back';
if (authRef) {
@ -1103,6 +1131,7 @@ async function promptForInteractiveSource(
const subpath = await promptText(prompts, {
message: sourceSubpathPrompt(source),
placeholder: 'optional',
...(currentState.sourceSubpath ? { initialValue: currentState.sourceSubpath } : {}),
});
if (subpath === undefined) return 'back';
if (subpath) {
@ -1121,7 +1150,10 @@ async function promptForInteractiveSource(
return await runSourcePromptSteps(initialState, () => [
...connectionSteps,
async (state) => {
const sourceUrl = await promptText(prompts, { message: 'Metabase URL' });
const sourceUrl = await promptText(prompts, {
message: 'Metabase URL',
...(state.sourceUrl ? { initialValue: state.sourceUrl } : {}),
});
if (sourceUrl === undefined) return 'back';
state.sourceUrl = sourceUrl;
return 'next';
@ -1133,6 +1165,7 @@ async function promptForInteractiveSource(
label: 'Metabase API key',
envName: 'METABASE_API_KEY',
secretFileName: `${state.sourceConnectionId ?? 'metabase-main'}-api-key`,
existingRef: state.sourceApiKeyRef,
});
if (ref === 'back') return 'back';
state.sourceApiKeyRef = ref;
@ -1164,13 +1197,19 @@ async function promptForInteractiveSource(
return await runSourcePromptSteps(initialState, () => [
...connectionSteps,
async (state) => {
const sourceUrl = await promptText(prompts, { message: 'Looker base URL' });
const sourceUrl = await promptText(prompts, {
message: 'Looker base URL',
...(state.sourceUrl ? { initialValue: state.sourceUrl } : {}),
});
if (sourceUrl === undefined) return 'back';
state.sourceUrl = sourceUrl;
return 'next';
},
async (state) => {
const sourceClientId = await promptText(prompts, { message: 'Looker client id' });
const sourceClientId = await promptText(prompts, {
message: 'Looker client id',
...(state.sourceClientId ? { initialValue: state.sourceClientId } : {}),
});
if (sourceClientId === undefined) return 'back';
state.sourceClientId = sourceClientId;
return 'next';
@ -1182,6 +1221,7 @@ async function promptForInteractiveSource(
label: 'Looker client secret',
envName: 'LOOKER_CLIENT_SECRET',
secretFileName: `${state.sourceConnectionId ?? 'looker-main'}-client-secret`,
existingRef: state.sourceClientSecretRef,
});
if (ref === 'back') return 'back';
state.sourceClientSecretRef = ref;
@ -1200,6 +1240,7 @@ async function promptForInteractiveSource(
const lookerConnectionName = await promptText(prompts, {
message: 'Looker connection name',
placeholder: 'optional',
...(state.sourceTarget ? { initialValue: state.sourceTarget } : {}),
});
if (lookerConnectionName === undefined) return 'back';
if (lookerConnectionName) {
@ -1221,6 +1262,7 @@ async function promptForInteractiveSource(
label: 'Notion integration token',
envName: 'NOTION_TOKEN',
secretFileName: `${currentState.sourceConnectionId ?? 'notion-main'}-token`,
existingRef: currentState.sourceApiKeyRef,
});
if (ref === 'back') return 'back';
currentState.sourceApiKeyRef = ref;
@ -1285,6 +1327,24 @@ function existingConnectionIdsBySource(
.sort((left, right) => left.localeCompare(right));
}
function sourceTypeForConnection(connection: KtxProjectConnectionConfig): KtxSetupSourceType | null {
const driver = String(connection.driver ?? '').toLowerCase();
return SOURCE_OPTIONS.some((option) => option.value === driver) ? (driver as KtxSetupSourceType) : null;
}
function contextSourceEditTargets(connections: Record<string, KtxProjectConnectionConfig>): Array<{
connectionId: string;
source: KtxSetupSourceType;
}> {
return Object.entries(connections)
.map(([connectionId, connection]) => {
const source = sourceTypeForConnection(connection);
return source ? { connectionId, source } : null;
})
.filter((target): target is { connectionId: string; source: KtxSetupSourceType } => target !== null)
.sort((left, right) => left.connectionId.localeCompare(right.connectionId));
}
function sourceChecklistForConnections(connections: Record<string, KtxProjectConnectionConfig>): {
options: Array<{ value: KtxSetupSourceType; label: string; hint?: string }>;
initialValues: KtxSetupSourceType[];
@ -1316,6 +1376,180 @@ function defaultConnectionIdForSource(
return `${base}-${index}`;
}
function firstStringRecordEntry(value: unknown): [string, string] | undefined {
if (!isRecord(value)) return undefined;
for (const [key, raw] of Object.entries(value)) {
if (typeof raw === 'string' && raw.trim().length > 0) {
return [key, raw.trim()];
}
}
return undefined;
}
function applyRepoSourceArgs(
args: KtxSetupSourcesArgs,
input: { repoUrl?: string; sourceDir?: string; branch?: string; subpath?: string; authTokenRef?: string },
): void {
if (input.sourceDir) {
args.sourcePath = input.sourceDir;
} else if (input.repoUrl?.startsWith('file:')) {
args.sourcePath = fileURLToPath(input.repoUrl);
} else if (input.repoUrl) {
args.sourceGitUrl = input.repoUrl;
}
if (input.branch) args.sourceBranch = input.branch;
if (input.subpath) args.sourceSubpath = input.subpath;
if (input.authTokenRef) args.sourceAuthTokenRef = input.authTokenRef;
}
function sourceArgsFromExistingConnection(input: {
args: KtxSetupSourcesArgs;
source: KtxSetupSourceType;
connectionId: string;
connection: KtxProjectConnectionConfig;
}): KtxSetupSourcesArgs {
const sourceArgs: KtxSetupSourcesArgs = {
projectDir: input.args.projectDir,
inputMode: input.args.inputMode,
source: input.source,
sourceConnectionId: input.connectionId,
runInitialSourceIngest: input.args.runInitialSourceIngest,
skipSources: input.args.skipSources,
};
if (input.source === 'dbt') {
applyRepoSourceArgs(sourceArgs, {
sourceDir: stringField(input.connection.source_dir),
repoUrl: stringField(input.connection.repo_url),
branch: stringField(input.connection.branch),
subpath: stringField(input.connection.path),
authTokenRef: stringField(input.connection.auth_token_ref),
});
const profilesPath = stringField(input.connection.profiles_path);
const target = stringField(input.connection.target);
const projectName = stringField(input.connection.project_name);
if (profilesPath) sourceArgs.sourceProfilesPath = profilesPath;
if (target) sourceArgs.sourceTarget = target;
if (projectName) sourceArgs.sourceProjectName = projectName;
return sourceArgs;
}
if (input.source === 'metricflow') {
const metricflow = isRecord(input.connection.metricflow) ? input.connection.metricflow : {};
applyRepoSourceArgs(sourceArgs, {
repoUrl: stringField(metricflow.repoUrl),
branch: stringField(metricflow.branch),
subpath: stringField(metricflow.path),
authTokenRef: stringField(metricflow.auth_token_ref),
});
return sourceArgs;
}
if (input.source === 'lookml') {
applyRepoSourceArgs(sourceArgs, {
repoUrl: stringField(input.connection.repoUrl),
branch: stringField(input.connection.branch),
subpath: stringField(input.connection.path),
authTokenRef: stringField(input.connection.auth_token_ref),
});
const mappings = isRecord(input.connection.mappings) ? input.connection.mappings : {};
const expectedLookerConnectionName = stringField(mappings.expectedLookerConnectionName);
if (expectedLookerConnectionName) sourceArgs.sourceTarget = expectedLookerConnectionName;
return sourceArgs;
}
if (input.source === 'metabase') {
sourceArgs.sourceUrl = stringField(input.connection.api_url);
sourceArgs.sourceApiKeyRef = stringField(input.connection.api_key_ref);
const mappings = isRecord(input.connection.mappings) ? input.connection.mappings : {};
const databaseMapping = firstStringRecordEntry(mappings.databaseMappings);
if (databaseMapping) {
sourceArgs.metabaseDatabaseId = Number.parseInt(databaseMapping[0], 10);
sourceArgs.sourceWarehouseConnectionId = databaseMapping[1];
}
return sourceArgs;
}
if (input.source === 'looker') {
sourceArgs.sourceUrl = stringField(input.connection.base_url);
sourceArgs.sourceClientId = stringField(input.connection.client_id);
sourceArgs.sourceClientSecretRef = stringField(input.connection.client_secret_ref);
const mappings = isRecord(input.connection.mappings) ? input.connection.mappings : {};
const connectionMapping = firstStringRecordEntry(mappings.connectionMappings);
if (connectionMapping) {
sourceArgs.sourceTarget = connectionMapping[0];
sourceArgs.sourceWarehouseConnectionId = connectionMapping[1];
}
return sourceArgs;
}
sourceArgs.sourceApiKeyRef = stringField(input.connection.auth_token_ref);
sourceArgs.notionCrawlMode =
input.connection.crawl_mode === 'all_accessible' ? 'all_accessible' : 'selected_roots';
if (Array.isArray(input.connection.root_page_ids)) {
sourceArgs.notionRootPageIds = input.connection.root_page_ids.filter(
(pageId): pageId is string => typeof pageId === 'string',
);
}
return sourceArgs;
}
async function promptEditedSourceConnection(input: {
args: KtxSetupSourcesArgs;
source: KtxSetupSourceType;
connectionId: string;
connection: KtxProjectConnectionConfig;
prompts: KtxSetupSourcesPromptAdapter;
io: KtxCliIo;
testGitRepo?: KtxSetupSourcesDeps['testGitRepo'];
pickNotionRootPages?: KtxSetupSourcesDeps['pickNotionRootPages'];
discoverMetabaseDatabases?: KtxSetupSourcesDeps['discoverMetabaseDatabases'];
}): Promise<Extract<InteractiveSourceConnectionChoice, { kind: 'edited' }> | 'back'> {
const sourceArgs = await promptForInteractiveSource(
sourceArgsFromExistingConnection({
args: input.args,
source: input.source,
connectionId: input.connectionId,
connection: input.connection,
}),
input.source,
input.prompts,
input.io,
{
pickNotionRootPages: input.pickNotionRootPages,
discoverMetabaseDatabases: input.discoverMetabaseDatabases,
},
input.connectionId,
input.testGitRepo,
input.discoverMetabaseDatabases,
);
return sourceArgs === 'back'
? 'back'
: { kind: 'edited', connectionId: input.connectionId, args: sourceArgs };
}
async function chooseContextSourceToEdit(input: {
projectDir: string;
prompts: KtxSetupSourcesPromptAdapter;
}): Promise<{ connectionId: string; source: KtxSetupSourceType } | 'back'> {
const project = await loadKtxProject({ projectDir: input.projectDir });
const targets = contextSourceEditTargets(project.config.connections);
if (targets.length === 0) return 'back';
const choice = await input.prompts.select({
message: 'Context source to edit',
options: [
...targets.map((target) => ({
value: target.connectionId,
label: `${target.connectionId} (${sourceLabel(target.source)})`,
})),
{ value: 'back', label: 'Back' },
],
});
if (choice === 'back') return 'back';
const target = targets.find((candidate) => candidate.connectionId === choice);
return target ?? 'back';
}
async function chooseInteractiveSourceConnection(input: {
args: KtxSetupSourcesArgs;
source: KtxSetupSourceType;
@ -1355,6 +1589,10 @@ async function chooseInteractiveSourceConnection(input: {
value: `existing:${connectionId}`,
label: `Use existing ${label} connection: ${connectionId}`,
})),
...existingIds.map((connectionId) => ({
value: `edit:${connectionId}`,
label: `Edit existing ${label} connection: ${connectionId}`,
})),
{ value: 'new', label: `Add new ${label} connection` },
{ value: 'back', label: 'Back' },
],
@ -1368,6 +1606,28 @@ async function chooseInteractiveSourceConnection(input: {
}
continue;
}
if (choice.startsWith('edit:')) {
const connectionId = choice.slice('edit:'.length);
const connection = input.connections[connectionId];
if (!connection) {
continue;
}
const edited = await promptEditedSourceConnection({
args: input.args,
source: input.source,
connectionId,
connection,
prompts: input.prompts,
io: input.io,
testGitRepo: input.testGitRepo,
pickNotionRootPages: input.pickNotionRootPages,
discoverMetabaseDatabases: input.discoverMetabaseDatabases,
});
if (edited === 'back') {
continue;
}
return edited;
}
const sourceArgs = await promptForInteractiveSource(
input.args,
input.source,
@ -1432,6 +1692,85 @@ async function validateSource(
return await (deps.validateNotion ?? defaultValidateNotion)(args.connection);
}
async function saveValidateAndMaybeBuildSource(input: {
args: KtxSetupSourcesArgs;
source: KtxSetupSourceType;
sourceChoice: Exclude<InteractiveSourceConnectionChoice, 'back'>;
prompts: KtxSetupSourcesPromptAdapter;
io: KtxCliIo;
deps: KtxSetupSourcesDeps;
}): Promise<SourceSetupChoiceResult> {
const connectionId =
input.sourceChoice.kind === 'existing'
? input.sourceChoice.connectionId
: input.sourceChoice.kind === 'edited'
? input.sourceChoice.connectionId
: (input.sourceChoice.args.sourceConnectionId ?? `${input.source}-main`);
const connection =
input.sourceChoice.kind === 'existing'
? input.sourceChoice.connection
: buildConnection(input.source, input.sourceChoice.args);
const rollback =
input.sourceChoice.kind === 'existing'
? undefined
: await writeSourceConnection(
input.args.projectDir,
connectionId,
connection,
sourceAdapter(input.source),
);
if (input.sourceChoice.kind === 'existing') {
await ensureSourceAdapterEnabled(input.args.projectDir, input.source);
}
const validation = await validateSource(
input.source,
{ projectDir: input.args.projectDir, connectionId, connection },
input.deps,
);
if (!validation.ok) {
await rollback?.();
input.io.stderr.write(`${validation.message}\n`);
return { status: 'failed' };
}
if (input.source === 'metabase' || input.source === 'looker') {
input.prompts.log?.(`Validating ${sourceLabel(input.source)} mapping…`);
const mappingCode = await (input.deps.runMapping ?? defaultRunMapping)(
input.args.projectDir,
connectionId,
createSetupPrefixedIo(input.io),
);
if (mappingCode !== 0) {
await rollback?.();
return { status: 'failed' };
}
}
if (input.args.runInitialSourceIngest) {
const ingestResult = await runInitialSourceIngestWithRecovery({
args: input.args,
connectionId,
io: input.io,
prompts: input.prompts,
deps: input.deps,
});
if (ingestResult === 'failed') {
await rollback?.();
return { status: 'failed' };
}
if (ingestResult === 'back') {
await rollback?.();
return { status: 'back' };
}
} else {
input.io.stdout.write(`│ Context source ${connectionId} saved. It will be built during the context build step.\n`);
}
return { status: 'ready', connectionId };
}
export async function runKtxSetupSourcesStep(
args: KtxSetupSourcesArgs,
io: KtxCliIo,
@ -1509,62 +1848,27 @@ export async function runKtxSetupSourcesStep(
returnToSourceSelection = true;
break;
}
const connectionId =
sourceChoice.kind === 'existing'
? sourceChoice.connectionId
: (sourceChoice.args.sourceConnectionId ?? `${source}-main`);
const connection =
sourceChoice.kind === 'existing' ? sourceChoice.connection : buildConnection(source, sourceChoice.args);
const rollback =
sourceChoice.kind === 'existing'
? undefined
: await writeSourceConnection(args.projectDir, connectionId, connection, sourceAdapter(source));
if (sourceChoice.kind === 'existing') {
await ensureSourceAdapterEnabled(args.projectDir, source);
}
const validation = await validateSource(source, { projectDir: args.projectDir, connectionId, connection }, deps);
if (!validation.ok) {
await rollback?.();
io.stderr.write(`${validation.message}\n`);
const choiceResult = await saveValidateAndMaybeBuildSource({
args,
source,
sourceChoice,
prompts,
io,
deps,
});
if (choiceResult.status === 'failed') {
return { status: 'failed', projectDir: args.projectDir };
}
if (source === 'metabase' || source === 'looker') {
prompts.log?.(`Validating ${sourceLabel(source)} mapping…`);
const mappingCode = await (deps.runMapping ?? defaultRunMapping)(
args.projectDir,
connectionId,
createSetupPrefixedIo(io),
);
if (mappingCode !== 0) {
await rollback?.();
return { status: 'failed', projectDir: args.projectDir };
if (choiceResult.status === 'back') {
if (args.source) {
return { status: 'back', projectDir: args.projectDir };
}
returnToSourceSelection = true;
break;
}
if (args.runInitialSourceIngest) {
const ingestResult = await runInitialSourceIngestWithRecovery({
args,
connectionId,
io,
prompts,
deps,
});
if (ingestResult === 'failed') {
await rollback?.();
return { status: 'failed', projectDir: args.projectDir };
}
if (ingestResult === 'back') {
await rollback?.();
if (args.source) {
return { status: 'back', projectDir: args.projectDir };
}
returnToSourceSelection = true;
break;
}
} else {
io.stdout.write(`│ Context source ${connectionId} saved. It will be built during the context build step.\n`);
if (!readyConnectionIds.includes(choiceResult.connectionId)) {
readyConnectionIds.push(choiceResult.connectionId);
}
readyConnectionIds.push(connectionId);
}
if (returnToSourceSelection) {
@ -1572,14 +1876,66 @@ export async function runKtxSetupSourcesStep(
}
if (readyConnectionIds.length > 0 && !args.source && args.inputMode !== 'disabled') {
const addMore = await prompts.select({
message: `${readyConnectionIds.length} context source${readyConnectionIds.length > 1 ? 's' : ''} configured (${readyConnectionIds.join(', ')}). Add another?`,
options: [
{ value: 'done', label: 'Done — continue to context build' },
{ value: 'add', label: 'Add another context source' },
],
});
if (addMore === 'add') {
let restartSourceSelection = false;
while (true) {
const addMore = await prompts.select({
message: `${readyConnectionIds.length} context source${readyConnectionIds.length > 1 ? 's' : ''} configured (${readyConnectionIds.join(', ')}). Add another?`,
options: [
{ value: 'done', label: 'Done — continue to context build' },
{ value: 'edit', label: 'Edit an existing context source' },
{ value: 'add', label: 'Add another context source' },
],
});
if (addMore === 'add') {
restartSourceSelection = true;
break;
}
if (addMore === 'edit') {
const editTarget = await chooseContextSourceToEdit({ projectDir: args.projectDir, prompts });
if (editTarget === 'back') {
continue;
}
const projectForEdit = await loadKtxProject({ projectDir: args.projectDir });
const connection = projectForEdit.config.connections[editTarget.connectionId];
if (!connection) {
continue;
}
const sourceChoice = await promptEditedSourceConnection({
args,
source: editTarget.source,
connectionId: editTarget.connectionId,
connection,
prompts,
io,
testGitRepo: deps.testGitRepo,
pickNotionRootPages: deps.pickNotionRootPages,
discoverMetabaseDatabases: deps.discoverMetabaseDatabases,
});
if (sourceChoice === 'back') {
continue;
}
const choiceResult = await saveValidateAndMaybeBuildSource({
args,
source: editTarget.source,
sourceChoice,
prompts,
io,
deps,
});
if (choiceResult.status === 'failed') {
return { status: 'failed', projectDir: args.projectDir };
}
if (choiceResult.status === 'back') {
continue;
}
if (!readyConnectionIds.includes(choiceResult.connectionId)) {
readyConnectionIds.push(choiceResult.connectionId);
}
continue;
}
break;
}
if (restartSourceSelection) {
continue;
}
}

667
packages/cli/src/status-project.ts Normal file
View file

@ -0,0 +1,667 @@
import type {
KtxLocalProject,
KtxProjectConfig,
KtxProjectConnectionConfig,
KtxProjectEmbeddingConfig,
KtxProjectLlmConfig,
} from '@ktx/context/project';
import type { DoctorCheck } from './doctor.js';
type ProjectStatusLevel = 'ok' | 'warn' | 'fail';
type ProjectVerdict = 'ready' | 'partial' | 'blocked';
interface ProjectStatusLine {
status: ProjectStatusLevel;
detail: string;
fix?: string;
}
interface LlmStatus extends ProjectStatusLine {
backend: string;
model?: string;
}
interface EmbeddingsStatus extends ProjectStatusLine {
backend: string;
model?: string;
dimensions?: number;
}
interface ConnectionStatus extends ProjectStatusLine {
name: string;
driver: string;
}
interface PipelineStatus {
adapters: string[];
enrichmentMode: string;
relationshipsEnabled: boolean;
relationshipsLlmProposals: boolean;
relationshipsValidationRequired: boolean;
agentEnabled: boolean;
agentTools: string[];
agentMaxIterations: number;
}
interface StorageStatus {
state: string;
search: string;
gitAutoCommit: boolean;
gitAuthor: string;
}
interface WarningItem {
message: string;
fix?: string;
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value !== null && !Array.isArray(value);
}
function hasOwnField(value: Record<string, unknown>, key: string): boolean {
return Object.prototype.hasOwnProperty.call(value, key);
}
export interface ProjectStatus {
projectName: string;
projectDir: string;
llm: LlmStatus;
embeddings: EmbeddingsStatus;
storage: StorageStatus;
connections: ConnectionStatus[];
pipeline: PipelineStatus;
warnings: WarningItem[];
verdict: ProjectVerdict;
verdictReason: string;
nextActions: string[];
promptCaching?: { enabled: boolean; systemTtl?: string; toolsTtl?: string; historyTtl?: string };
workUnits?: { stepBudget: number; maxConcurrency: number; failureMode: string };
memoryAutoCommit: boolean;
relationshipsDetail?: {
acceptThreshold: number;
reviewThreshold: number;
maxLlmTablesPerBatch: number;
validationConcurrency: number;
};
}
function resolveRef(value: unknown, env: NodeJS.ProcessEnv): { resolved: string; via: 'literal' | 'env' | 'file' | 'missing' } {
if (typeof value !== 'string') return { resolved: '', via: 'missing' };
const trimmed = value.trim();
if (trimmed.length === 0) return { resolved: '', via: 'missing' };
if (trimmed.startsWith('env:')) {
const name = trimmed.slice(4).trim();
const v = env[name];
return v && v.trim().length > 0 ? { resolved: v, via: 'env' } : { resolved: '', via: 'missing' };
}
if (trimmed.startsWith('file:')) {
return { resolved: trimmed.slice(5), via: 'file' };
}
return { resolved: trimmed, via: 'literal' };
}
function envHint(value: unknown): string | undefined {
if (typeof value === 'string' && value.trim().startsWith('env:')) {
return value.trim().slice(4).trim();
}
return undefined;
}
function buildLlmStatus(config: KtxProjectLlmConfig, env: NodeJS.ProcessEnv): LlmStatus {
const backend = config.provider.backend;
const model = config.models?.default;
if (backend === 'none') {
return {
backend,
model,
status: 'fail',
detail: 'no LLM configured — ktx ask will not work',
fix: 'Run: ktx setup (choose an LLM provider)',
};
}
if (backend === 'anthropic') {
const ref = config.provider.anthropic?.api_key;
const resolved = resolveRef(ref, env);
if (resolved.resolved.length > 0) {
return { backend, model, status: 'ok', detail: `key set${resolved.via === 'env' ? ` (env)` : ''}` };
}
if (env.ANTHROPIC_API_KEY && env.ANTHROPIC_API_KEY.trim().length > 0) {
return { backend, model, status: 'ok', detail: 'key set (env: ANTHROPIC_API_KEY)' };
}
const hint = envHint(ref);
return {
backend,
model,
status: 'warn',
detail: hint ? `key missing (env: ${hint})` : 'key missing',
fix: hint ? `Set ${hint}` : 'Set ANTHROPIC_API_KEY or rerun `ktx setup`',
};
}
if (backend === 'vertex') {
const project = config.provider.vertex?.project;
if (project && project.length > 0) {
return { backend, model, status: 'ok', detail: `project=${project}` };
}
return { backend, model, status: 'warn', detail: 'vertex project not configured', fix: 'Rerun `ktx setup`' };
}
if (backend === 'gateway') {
const ref = config.provider.gateway?.api_key;
const resolved = resolveRef(ref, env);
if (resolved.resolved.length > 0) {
return { backend, model, status: 'ok', detail: 'key set' };
}
const hint = envHint(ref);
return {
backend,
model,
status: 'warn',
detail: hint ? `key missing (env: ${hint})` : 'key missing',
fix: hint ? `Set ${hint}` : 'Set the gateway api_key or rerun `ktx setup`',
};
}
return { backend, model, status: 'warn', detail: 'unknown LLM backend' };
}
function buildEmbeddingsStatus(config: KtxProjectEmbeddingConfig, env: NodeJS.ProcessEnv): EmbeddingsStatus {
const backend = config.backend;
const model = config.model;
const dimensions = config.dimensions;
if (backend === 'none') {
return {
backend,
model,
dimensions,
status: 'warn',
detail: 'disabled — semantic search will be skipped',
};
}
if (backend === 'deterministic') {
return {
backend,
model,
dimensions,
status: 'warn',
detail: 'deterministic — semantic search degraded (lexical/dictionary lanes still work)',
};
}
if (backend === 'openai') {
const ref = config.openai?.api_key;
const resolved = resolveRef(ref, env);
if (resolved.resolved.length > 0 || (env.OPENAI_API_KEY && env.OPENAI_API_KEY.trim().length > 0)) {
return { backend, model, dimensions, status: 'ok', detail: 'key set' };
}
const hint = envHint(ref);
return {
backend,
model,
dimensions,
status: 'warn',
detail: hint ? `key missing (env: ${hint})` : 'key missing',
fix: hint ? `Set ${hint}` : 'Set OPENAI_API_KEY or rerun `ktx setup`',
};
}
if (backend === 'sentence-transformers') {
const url = config.sentenceTransformers?.base_url;
if (typeof url === 'string' && url.length > 0) {
return { backend, model, dimensions, status: 'ok', detail: `service: ${url}` };
}
return {
backend,
model,
dimensions,
status: 'warn',
detail: 'no base_url configured',
fix: 'Rerun `ktx setup`',
};
}
return { backend, model, dimensions, status: 'warn', detail: 'unknown embedding backend' };
}
function buildConnectionStatus(
name: string,
conn: KtxProjectConnectionConfig,
env: NodeJS.ProcessEnv,
): ConnectionStatus {
const driver = (conn.driver ?? 'unknown').toLowerCase();
const ok = (detail: string): ConnectionStatus => ({ name, driver, status: 'ok', detail });
const warn = (detail: string, fix?: string): ConnectionStatus => ({ name, driver, status: 'warn', detail, fix });
switch (driver) {
case 'postgres':
case 'postgresql':
case 'mysql':
case 'clickhouse':
case 'sqlserver': {
const urlRef = resolveRef(conn.url, env);
if (urlRef.resolved.length > 0) return ok(`url configured`);
if (typeof (conn as Record<string, unknown>).host === 'string') return ok('host configured');
const hint = envHint(conn.url);
return warn(hint ? `url missing (env: ${hint})` : 'url not set', hint ? `Set ${hint}` : 'Rerun `ktx setup`');
}
case 'snowflake': {
const account = (conn as Record<string, unknown>).account;
if (typeof account === 'string' && account.length > 0) return ok(`account: ${account}`);
return warn('account not set', 'Rerun `ktx setup`');
}
case 'bigquery': {
const cred = resolveRef((conn as Record<string, unknown>).credentials_json, env);
if (cred.resolved.length > 0) return ok('credentials configured');
const hint = envHint((conn as Record<string, unknown>).credentials_json);
return warn(hint ? `credentials missing (env: ${hint})` : 'credentials not set', hint ? `Set ${hint}` : 'Rerun `ktx setup`');
}
case 'sqlite': {
const path = (conn as Record<string, unknown>).path;
if (typeof path === 'string' && path.length > 0) return ok(`path: ${path}`);
return warn('path not set', 'Rerun `ktx setup`');
}
case 'notion': {
const tokenRef =
(conn as Record<string, unknown>).auth_token_ref ??
(conn as Record<string, unknown>).auth_token;
const resolved = resolveRef(tokenRef, env);
if (resolved.resolved.length > 0) return ok('auth token configured');
const hint = envHint(tokenRef);
return warn(hint ? `auth token missing (env: ${hint})` : 'auth token not set', hint ? `Set ${hint}` : 'Rerun `ktx setup`');
}
case 'dbt':
case 'dbt-core':
case 'dbt-cloud': {
const repoUrl =
(conn as Record<string, unknown>).repoUrl ??
(conn as Record<string, unknown>).repo_url;
if (typeof repoUrl === 'string' && repoUrl.length > 0) return ok(`repo: ${repoUrl}`);
return warn('repoUrl not set', 'Rerun `ktx setup`');
}
case 'metabase': {
const url = (conn as Record<string, unknown>).url ?? (conn as Record<string, unknown>).base_url;
if (typeof url === 'string' && url.length > 0) return ok(`url: ${url}`);
return warn('url not set', 'Rerun `ktx setup`');
}
case 'looker':
case 'lookml': {
const url = (conn as Record<string, unknown>).base_url ?? (conn as Record<string, unknown>).url;
if (typeof url === 'string' && url.length > 0) return ok(`url: ${url}`);
return warn('base_url not set', 'Rerun `ktx setup`');
}
case 'metricflow': {
const repoUrl = (conn as Record<string, unknown>).repoUrl ?? (conn as Record<string, unknown>).repo_url;
if (typeof repoUrl === 'string' && repoUrl.length > 0) return ok(`repo: ${repoUrl}`);
return warn('repoUrl not set', 'Rerun `ktx setup`');
}
default:
return { name, driver, status: 'ok', detail: 'configured' };
}
}
const ADAPTER_DRIVER_REQUIREMENT: Record<string, string[]> = {
'live-database': ['postgres', 'postgresql', 'mysql', 'snowflake', 'bigquery', 'clickhouse', 'sqlite', 'sqlserver'],
dbt: ['dbt', 'dbt-core', 'dbt-cloud'],
notion: ['notion'],
metabase: ['metabase'],
looker: ['looker', 'lookml'],
lookml: ['looker', 'lookml'],
metricflow: ['metricflow'],
};
function buildPipelineStatus(config: KtxProjectConfig): PipelineStatus {
return {
adapters: config.ingest.adapters,
enrichmentMode: config.scan.enrichment.mode,
relationshipsEnabled: config.scan.relationships.enabled,
relationshipsLlmProposals: config.scan.relationships.llmProposals,
relationshipsValidationRequired: config.scan.relationships.validationRequiredForManifest,
agentEnabled: config.agent.run_research.enabled,
agentTools: config.agent.run_research.default_toolset,
agentMaxIterations: config.agent.run_research.max_iterations,
};
}
function buildStorageStatus(config: KtxProjectConfig): StorageStatus {
return {
state: config.storage.state,
search: config.storage.search,
gitAutoCommit: config.storage.git.auto_commit,
gitAuthor: config.storage.git.author,
};
}
function buildWarnings(
config: KtxProjectConfig,
connections: ConnectionStatus[],
llm: LlmStatus,
embeddings: EmbeddingsStatus,
): WarningItem[] {
const warnings: WarningItem[] = [];
for (const [connectionId, connection] of Object.entries(config.connections)) {
const driver = String(connection.driver ?? '').toLowerCase();
if (hasOwnField(connection, 'readonly')) {
warnings.push({
message: `connections.${connectionId}.readonly is no longer used.`,
fix: `Remove connections.${connectionId}.readonly from ktx.yaml.`,
});
}
if ((driver === 'sqlite' || driver === 'sqlite3') && hasOwnField(connection, 'file_path')) {
warnings.push({
message: `connections.${connectionId}.file_path was removed.`,
fix: `Rename connections.${connectionId}.file_path to path.`,
});
}
if (driver === 'notion' && hasOwnField(connection, 'last_successful_cursor')) {
warnings.push({
message: `connections.${connectionId}.last_successful_cursor is local sync state.`,
fix: 'Remove it from ktx.yaml. KTX stores the Notion cursor in .ktx/db.sqlite.',
});
}
const historicSql = isRecord(connection.historicSql) ? connection.historicSql : null;
if (!historicSql) {
continue;
}
if (hasOwnField(historicSql, 'concurrency')) {
warnings.push({
message: `connections.${connectionId}.historicSql.concurrency is no longer used.`,
fix: `Remove connections.${connectionId}.historicSql.concurrency from ktx.yaml.`,
});
}
const historicDialect = String(historicSql.dialect ?? driver).toLowerCase();
if (
(historicDialect === 'postgres' || historicDialect === 'postgresql') &&
hasOwnField(historicSql, 'windowDays')
) {
warnings.push({
message: `connections.${connectionId}.historicSql.windowDays does not constrain pg_stat_statements.`,
fix: `Remove connections.${connectionId}.historicSql.windowDays from ktx.yaml.`,
});
}
}
for (const adapter of config.ingest.adapters) {
const requiredDrivers = ADAPTER_DRIVER_REQUIREMENT[adapter];
if (!requiredDrivers) continue;
const hasMatching = connections.some((c) => requiredDrivers.includes(c.driver));
if (!hasMatching) {
warnings.push({
message: `Adapter "${adapter}" is enabled but no connection of type ${requiredDrivers.slice(0, 2).join('/')} is configured.`,
fix: 'Rerun `ktx setup` to add a connection, or remove the adapter from ingest.adapters.',
});
}
}
if (config.agent.run_research.enabled && llm.backend === 'none') {
warnings.push({
message: 'Research agent is enabled but LLM is not configured.',
fix: 'Set up an LLM provider via `ktx setup` or disable agent.run_research.enabled.',
});
}
if (embeddings.backend === 'none' && config.ingest.adapters.includes('live-database')) {
warnings.push({
message: 'Semantic search is off (embeddings backend = none). Lexical/dictionary lanes still work.',
});
}
return warnings;
}
function buildVerdict(
llm: LlmStatus,
embeddings: EmbeddingsStatus,
connections: ConnectionStatus[],
warnings: WarningItem[],
): { verdict: ProjectVerdict; reason: string; nextActions: string[] } {
if (llm.status === 'fail') {
return {
verdict: 'blocked',
reason: 'LLM not configured — `ktx ask` will not work.',
nextActions: ['ktx setup'],
};
}
const reasons: string[] = [];
if (llm.status === 'warn') reasons.push('LLM credentials missing');
if (embeddings.status === 'warn') {
if (embeddings.backend === 'deterministic' || embeddings.backend === 'none') {
reasons.push('semantic search disabled');
} else {
reasons.push('embedding credentials missing');
}
}
const missing = connections.filter((c) => c.status !== 'ok').length;
if (missing > 0) reasons.push(`${missing} connection${missing === 1 ? '' : 's'} need configuration`);
if (warnings.length > 0) reasons.push(`${warnings.length} config warning${warnings.length === 1 ? '' : 's'}`);
if (reasons.length === 0) {
return {
verdict: 'ready',
reason: 'Ready.',
nextActions: ['ktx scan', 'ktx wiki', 'ktx sl ask "…"'],
};
}
return {
verdict: 'partial',
reason: `Partially ready — ${reasons.join('; ')}.`,
nextActions: ['ktx setup'],
};
}
export interface BuildProjectStatusOptions {
env?: NodeJS.ProcessEnv;
}
export function buildProjectStatus(project: KtxLocalProject, options: BuildProjectStatusOptions = {}): ProjectStatus {
const env = options.env ?? process.env;
const config = project.config;
const llm = buildLlmStatus(config.llm, env);
const embeddings = buildEmbeddingsStatus(config.ingest.embeddings, env);
const storage = buildStorageStatus(config);
const connections = Object.entries(config.connections).map(([name, conn]) =>
buildConnectionStatus(name, conn, env),
);
const pipeline = buildPipelineStatus(config);
const warnings = buildWarnings(config, connections, llm, embeddings);
const { verdict, reason, nextActions } = buildVerdict(llm, embeddings, connections, warnings);
return {
projectName: config.project,
projectDir: project.projectDir,
llm,
embeddings,
storage,
connections,
pipeline,
warnings,
verdict,
verdictReason: reason,
nextActions,
promptCaching: config.llm.promptCaching
? {
enabled: config.llm.promptCaching.enabled ?? false,
systemTtl: config.llm.promptCaching.systemTtl,
toolsTtl: config.llm.promptCaching.toolsTtl,
historyTtl: config.llm.promptCaching.historyTtl,
}
: undefined,
workUnits: {
stepBudget: config.ingest.workUnits.stepBudget,
maxConcurrency: config.ingest.workUnits.maxConcurrency,
failureMode: config.ingest.workUnits.failureMode,
},
memoryAutoCommit: config.memory.auto_commit,
relationshipsDetail: {
acceptThreshold: config.scan.relationships.acceptThreshold,
reviewThreshold: config.scan.relationships.reviewThreshold,
maxLlmTablesPerBatch: config.scan.relationships.maxLlmTablesPerBatch,
validationConcurrency: config.scan.relationships.validationConcurrency,
},
};
}
// ─── Rendering ──────────────────────────────────────────────────────────────
const SYMBOL: Record<ProjectStatusLevel, string> = { ok: '✓', warn: '⚠', fail: '✗' };
function ansi(useColor: boolean, code: string, text: string, closer = '39'): string {
return useColor ? `\u001b[${code}m${text}\u001b[${closer}m` : text;
}
function colorFor(level: ProjectStatusLevel): string {
return level === 'ok' ? '32' : level === 'warn' ? '33' : '31';
}
function abbreviateHome(filePath: string, env: NodeJS.ProcessEnv): string {
const home = env.HOME;
if (home && (filePath === home || filePath.startsWith(`${home}/`))) {
return filePath === home ? '~' : `~${filePath.slice(home.length)}`;
}
return filePath;
}
export interface RenderProjectStatusOptions {
verbose?: boolean;
useColor?: boolean;
durationMs?: number;
toolchainChecks?: DoctorCheck[];
env?: NodeJS.ProcessEnv;
}
export function renderProjectStatus(status: ProjectStatus, options: RenderProjectStatusOptions = {}): string {
const verbose = options.verbose ?? false;
const useColor = options.useColor ?? false;
const env = options.env ?? process.env;
const dim = (s: string) => ansi(useColor, '2', s, '22');
const bold = (s: string) => ansi(useColor, '1', s, '22');
const color = (level: ProjectStatusLevel, s: string) => ansi(useColor, colorFor(level), s);
const sym = (level: ProjectStatusLevel) => color(level, SYMBOL[level]);
const lines: string[] = [];
const dirStr = abbreviateHome(status.projectDir, env);
lines.push(`${bold('KTX status')} ${dim('·')} ${status.projectName} ${dim(`(${dirStr})`)}`);
lines.push('');
const labelPad = 'Connections'.length;
const label = (text: string) => text.padEnd(labelPad);
// Core readiness rows
const llmDetail = [status.llm.backend, status.llm.model].filter(Boolean).join(` ${dim('·')} `);
lines.push(` ${label('LLM')} ${llmDetail} ${sym(status.llm.status)} ${dim(status.llm.detail)}`);
const embedParts = [status.embeddings.backend];
if (status.embeddings.model) embedParts.push(status.embeddings.model);
const embedDim = status.embeddings.dimensions ? `(${status.embeddings.dimensions}d)` : '';
const embedDetail = `${embedParts.join(` ${dim('·')} `)}${embedDim ? ` ${embedDim}` : ''}`;
lines.push(` ${label('Embeddings')} ${embedDetail} ${sym(status.embeddings.status)} ${dim(status.embeddings.detail)}`);
lines.push(` ${label('Storage')} ${dim(`${status.storage.state} (state) · ${status.storage.search} (search)`)}`);
lines.push('');
// Connections
if (status.connections.length === 0) {
lines.push(` ${bold('Connections')} ${dim('(none)')}`);
lines.push(` ${dim('No connections configured. Run `ktx setup` to add one.')}`);
} else {
lines.push(` ${bold('Connections')} ${dim(`(${status.connections.length})`)}`);
const nameWidth = Math.max(...status.connections.map((c) => c.name.length));
const driverWidth = Math.max(...status.connections.map((c) => c.driver.length));
for (const conn of status.connections) {
lines.push(
` ${sym(conn.status)} ${conn.name.padEnd(nameWidth)} ${dim(conn.driver.padEnd(driverWidth))} ${conn.detail}`,
);
if (conn.fix && conn.status !== 'ok') {
const indent = 6 + nameWidth + 3 + driverWidth + 3;
lines.push(`${' '.repeat(indent)}${dim(`→ ${conn.fix}`)}`);
}
}
}
lines.push('');
// Pipeline
lines.push(` ${bold('Pipeline')}`);
const pipelineLabelWidth = Math.max('Adapters'.length, 'Enrichment'.length, 'Research agent'.length);
const pLabel = (text: string) => text.padEnd(pipelineLabelWidth);
lines.push(` ${pLabel('Adapters')} ${status.pipeline.adapters.length > 0 ? status.pipeline.adapters.join(', ') : dim('(none)')}`);
const enrichmentDetail = [`${status.pipeline.enrichmentMode} mode`];
if (status.pipeline.relationshipsEnabled) {
const bits = ['relationships on'];
if (status.pipeline.relationshipsLlmProposals) bits.push('LLM proposals');
if (status.pipeline.relationshipsValidationRequired) bits.push('validation required');
enrichmentDetail.push(bits.join(', '));
} else {
enrichmentDetail.push('relationships off');
}
lines.push(` ${pLabel('Enrichment')} ${enrichmentDetail.join(` ${dim('·')} `)}`);
const agentDetail = status.pipeline.agentEnabled
? `enabled ${dim(`(${status.pipeline.agentTools.length} tool${status.pipeline.agentTools.length === 1 ? '' : 's'})`)}`
: dim('disabled');
lines.push(` ${pLabel('Research agent')} ${agentDetail}`);
lines.push('');
// Warnings
if (status.warnings.length > 0) {
lines.push(` ${bold('Warnings')}`);
for (const w of status.warnings) {
lines.push(` ${color('warn', SYMBOL.warn)} ${w.message}`);
if (w.fix) lines.push(` ${dim(`→ ${w.fix}`)}`);
}
lines.push('');
}
// Verbose extras
if (verbose) {
if (options.toolchainChecks && options.toolchainChecks.length > 0) {
lines.push(` ${bold('Toolchain')}`);
for (const check of options.toolchainChecks) {
const lv: ProjectStatusLevel = check.status === 'pass' ? 'ok' : check.status === 'warn' ? 'warn' : 'fail';
lines.push(` ${sym(lv)} ${check.label}: ${check.detail}`);
if (check.fix && lv !== 'ok') lines.push(` ${dim(`→ ${check.fix}`)}`);
}
lines.push('');
}
if (status.promptCaching) {
const pc = status.promptCaching;
const bits = [`enabled=${pc.enabled}`];
if (pc.systemTtl) bits.push(`system=${pc.systemTtl}`);
if (pc.toolsTtl) bits.push(`tools=${pc.toolsTtl}`);
if (pc.historyTtl) bits.push(`history=${pc.historyTtl}`);
lines.push(` ${bold('Prompt caching')} ${dim(bits.join(', '))}`);
}
if (status.workUnits) {
const wu = status.workUnits;
lines.push(` ${bold('Work units')} ${dim(`stepBudget=${wu.stepBudget}, maxConcurrency=${wu.maxConcurrency}, failureMode=${wu.failureMode}`)}`);
}
if (status.relationshipsDetail) {
const r = status.relationshipsDetail;
lines.push(
` ${bold('Relationships')} ${dim(`accept=${r.acceptThreshold}, review=${r.reviewThreshold}, maxLlmTables=${r.maxLlmTablesPerBatch}, concurrency=${r.validationConcurrency}`)}`,
);
}
lines.push(
` ${bold('Agent')} ${dim(`max_iterations=${status.pipeline.agentMaxIterations}, tools=${status.pipeline.agentTools.join(', ') || '(none)'}`)}`,
);
lines.push(` ${bold('Memory')} ${dim(`auto_commit=${status.memoryAutoCommit}`)}`);
lines.push(
` ${bold('Git')} ${dim(`auto_commit=${status.storage.gitAutoCommit}, author=${status.storage.gitAuthor}`)}`,
);
lines.push('');
}
// Verdict + next steps
const verdictLevel: ProjectStatusLevel =
status.verdict === 'ready' ? 'ok' : status.verdict === 'partial' ? 'warn' : 'fail';
const duration = options.durationMs !== undefined ? ` ${dim(`(${(options.durationMs / 1000).toFixed(2)}s)`)}` : '';
if (status.verdict === 'ready') {
const hint = ` ${dim('Try:')} ${status.nextActions.join(dim(' · '))}`;
lines.push(`${color(verdictLevel, status.verdictReason)}${hint}${duration}`);
} else {
const hint = status.nextActions.length > 0 ? ` ${dim('Next:')} ${status.nextActions.join(dim(' · '))}` : '';
lines.push(`${color(verdictLevel, status.verdictReason)}${hint}${duration}`);
}
lines.push('');
return lines.join('\n');
}

112
packages/cli/src/notion-page-picker-tree.test.ts → packages/cli/src/tree-picker-state.test.ts
View file

@ -11,10 +11,9 @@ import {
selectAllVisible,
selectNone,
toggleChecked,
TRANSIENT_HINT_DURATION_MS,
visibleNodeIds,
type NotionPickerPageInput,
} from './notion-page-picker-tree.js';
type TreePickerNodeInput,
} from './tree-picker-state.js';
const IDS = {
engineering: '11111111-1111-1111-1111-111111111111',
@ -28,7 +27,7 @@ const IDS = {
cycleB: '99999999-9999-9999-9999-999999999999',
};
function pages(): NotionPickerPageInput[] {
function pages(): TreePickerNodeInput[] {
return [
{ id: IDS.marketing, title: 'Marketing', archived: false, parentId: null },
{ id: IDS.onboarding, title: 'Onboarding', archived: false, parentId: IDS.engineering },
@ -44,7 +43,7 @@ function pages(): NotionPickerPageInput[] {
}
describe('buildPickerTree', () => {
it('deduplicates pages, sorts siblings, preserves archived flags, roots orphans, and breaks cycles', () => {
it('deduplicates nodes, sorts siblings, preserves archived flags, roots orphans, and breaks cycles', () => {
const tree = buildPickerTree(pages());
const byId = new Map(tree.map((node) => [node.id, node]));
@ -90,8 +89,7 @@ describe('selection invariants', () => {
it('checking a parent locks descendants and keeps checked ids minimal', () => {
const state = buildInitialState({
tree: buildPickerTree(pages()),
existingRootPageIds: [],
currentCrawlMode: 'selected_roots',
existingSelectedIds: [],
});
const checkedParent = toggleChecked(state, IDS.engineering, 1000);
@ -113,15 +111,11 @@ describe('selection invariants', () => {
expect(canToggle(IDS.architecture, uncheckedParent)).toEqual({ ok: true });
});
it('normalizes stored roots, reports stale roots, expands checked ancestors, and flattens descendants', () => {
it('reports stale stored ids via the caller-supplied warning, expands checked ancestors, and flattens descendants', () => {
const state = buildInitialState({
tree: buildPickerTree(pages()),
existingRootPageIds: [
IDS.engineering.replaceAll('-', ''),
IDS.architecture,
'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa',
],
currentCrawlMode: 'selected_roots',
existingSelectedIds: [IDS.engineering, IDS.architecture, 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa'],
staleWarning: (staleCount) => `${staleCount} stored root_page_ids no longer visible`,
});
expect([...state.checked]).toEqual([IDS.engineering]);
@ -130,14 +124,21 @@ describe('selection invariants', () => {
expect(state.preLoadWarnings).toEqual(['1 stored root_page_ids no longer visible']);
expect(flattenSelection(new Set([IDS.engineering, IDS.architecture]), state.byId)).toEqual([IDS.engineering]);
});
it('falls back to a generic stale warning when no warning factory is supplied', () => {
const state = buildInitialState({
tree: buildPickerTree(pages()),
existingSelectedIds: ['aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa'],
});
expect(state.preLoadWarnings).toEqual(['1 stored selections no longer visible']);
});
});
describe('search and cursor movement', () => {
it('filters by title and path while deriving auto-expanded ancestors', () => {
const state = buildInitialState({
tree: buildPickerTree(pages()),
existingRootPageIds: [],
currentCrawlMode: 'selected_roots',
existingSelectedIds: [],
});
const searching = {
...state,
@ -154,8 +155,7 @@ describe('search and cursor movement', () => {
it('moves the cursor through visible nodes and implements left/right tree semantics', () => {
const state = buildInitialState({
tree: buildPickerTree(pages()),
existingRootPageIds: [],
currentCrawlMode: 'selected_roots',
existingSelectedIds: [],
});
const atEngineering = {
@ -176,8 +176,7 @@ describe('bulk actions and reducer effects', () => {
it('selects only matching visible roots under search and clears selection', () => {
const state = buildInitialState({
tree: buildPickerTree(pages()),
existingRootPageIds: [IDS.marketing],
currentCrawlMode: 'selected_roots',
existingSelectedIds: [IDS.marketing],
});
const searching = {
...state,
@ -189,56 +188,56 @@ describe('bulk actions and reducer effects', () => {
expect([...selectNone(selected).checked]).toEqual([]);
});
it('returns save immediately for selected_roots and requires confirmation for all_accessible', () => {
const selectedRoots = toggleChecked(
it('saves immediately when confirm is not required and prompts confirmation when requireConfirmOnSave is true', () => {
const noConfirm = toggleChecked(
buildInitialState({
tree: buildPickerTree(pages()),
existingRootPageIds: [],
currentCrawlMode: 'selected_roots',
existingSelectedIds: [],
}),
IDS.marketing,
1000,
);
expect(reducer(selectedRoots, 'save-request')).toEqual({
next: selectedRoots,
expect(reducer(noConfirm, 'save-request')).toEqual({
next: noConfirm,
effect: 'save',
});
const allAccessible = {
...selectedRoots,
currentCrawlMode: 'all_accessible' as const,
const confirmRequired = {
...noConfirm,
requireConfirmOnSave: true,
};
const confirm = reducer(allAccessible, 'save-request');
const confirm = reducer(confirmRequired, 'save-request');
expect(confirm).toEqual({
next: { ...allAccessible, pendingConfirm: 'mode-switch' },
next: { ...confirmRequired, pendingConfirm: 'save-confirm' },
effect: null,
});
expect(reducer(confirm.next, 'save-cancel')).toEqual({
next: { ...allAccessible, pendingConfirm: null },
next: { ...confirmRequired, pendingConfirm: null },
effect: null,
});
expect(reducer(confirm.next, 'save-confirm')).toEqual({
next: { ...allAccessible, pendingConfirm: null },
next: { ...confirmRequired, pendingConfirm: null },
effect: 'save',
});
});
it('blocks empty saves, updates search state, and quits without saving', () => {
it('prompts skip-empty confirmation on empty save, updates search state, and quits without saving', () => {
const state = buildInitialState({
tree: buildPickerTree(pages()),
existingRootPageIds: [],
currentCrawlMode: 'selected_roots',
existingSelectedIds: [],
});
const blockedSave = reducer(state, 'save-request', 9000);
expect(blockedSave).toEqual({
next: {
...state,
transientHint: {
text: 'Select at least one page or press q to quit',
expiresAt: 9000 + TRANSIENT_HINT_DURATION_MS,
},
},
const emptySave = reducer(state, 'save-request');
expect(emptySave).toEqual({
next: { ...state, pendingConfirm: 'skip-empty' },
effect: null,
});
expect(reducer(emptySave.next, 'save-confirm')).toEqual({
next: { ...state, pendingConfirm: null },
effect: 'quit-without-save',
});
expect(reducer(emptySave.next, 'save-cancel')).toEqual({
next: { ...state, pendingConfirm: null },
effect: null,
});
expect(
@ -253,16 +252,33 @@ describe('bulk actions and reducer effects', () => {
});
});
it('treats skip-empty confirmation as a save with empty selection when skipEmptyAction is save-empty', () => {
const state = buildInitialState({
tree: buildPickerTree(pages()),
existingSelectedIds: [],
skipEmptyAction: 'save-empty',
});
const emptySave = reducer(state, 'save-request');
expect(emptySave).toEqual({
next: { ...state, pendingConfirm: 'skip-empty' },
effect: null,
});
expect(reducer(emptySave.next, 'save-confirm')).toEqual({
next: { ...state, pendingConfirm: null },
effect: 'save',
});
});
it('clears transient hints only when their expiry time has passed', () => {
const state = buildInitialState({
tree: buildPickerTree(pages()),
existingRootPageIds: [],
currentCrawlMode: 'selected_roots',
existingSelectedIds: [],
});
const withHint = {
...state,
transientHint: {
text: 'Select at least one page or press q to quit',
text: 'Select at least one item or press esc to cancel',
expiresAt: 11500,
},
};

102
packages/cli/src/notion-page-picker-tree.ts → packages/cli/src/tree-picker-state.ts
View file

@ -1,11 +1,11 @@
export interface NotionPickerPageInput {
export interface TreePickerNodeInput {
id: string;
title?: string | null;
archived?: boolean;
parentId?: string | null;
}
interface NotionPickerNode {
export interface TreePickerNode {
id: string;
title: string;
archived: boolean;
@ -15,17 +15,22 @@ interface NotionPickerNode {
path: string;
}
type PendingConfirmKind = 'save-confirm' | 'skip-empty';
export type SkipEmptyAction = 'quit' | 'save-empty';
export interface PickerState {
tree: NotionPickerNode[];
byId: Map<string, NotionPickerNode>;
tree: TreePickerNode[];
byId: Map<string, TreePickerNode>;
expanded: Set<string>;
checked: Set<string>;
cursorId: string;
search: { editing: boolean; query: string };
pendingConfirm: 'mode-switch' | null;
pendingConfirm: PendingConfirmKind | null;
preLoadWarnings: string[];
transientHint: { text: string; expiresAt: number } | null;
currentCrawlMode: 'all_accessible' | 'selected_roots';
requireConfirmOnSave: boolean;
skipEmptyAction: SkipEmptyAction;
}
export type PickerCommand =
@ -61,29 +66,16 @@ interface MutableNode {
childIds: string[];
}
export const TRANSIENT_HINT_DURATION_MS = 2500;
const TRANSIENT_HINT_DURATION_MS = 2500;
const collator = new Intl.Collator('en', { sensitivity: 'base', numeric: true });
function normalizePageId(value: string): string {
const trimmed = value.trim();
const compact = trimmed.replace(/-/g, '');
if (/^[0-9a-fA-F]{32}$/.test(compact)) {
const lower = compact.toLowerCase();
return `${lower.slice(0, 8)}-${lower.slice(8, 12)}-${lower.slice(12, 16)}-${lower.slice(
16,
20,
)}-${lower.slice(20)}`;
}
return trimmed;
}
function titleValue(value: string | null | undefined): string {
const trimmed = value?.trim() ?? '';
return trimmed.length > 0 ? trimmed : 'Untitled';
}
function sortedNodeIds(ids: string[], nodes: Map<string, MutableNode | NotionPickerNode>): string[] {
function sortedNodeIds(ids: string[], nodes: Map<string, MutableNode | TreePickerNode>): string[] {
return [...ids].sort((leftId, rightId) => {
const left = nodes.get(leftId);
const right = nodes.get(rightId);
@ -107,7 +99,7 @@ export function clearExpiredTransientHint(state: PickerState, now = Date.now()):
return cloneState(state, { transientHint: null });
}
function ancestorsOf(nodeId: string, byId: Map<string, NotionPickerNode>): string[] {
function ancestorsOf(nodeId: string, byId: Map<string, TreePickerNode>): string[] {
const ancestors: string[] = [];
let parentId = byId.get(nodeId)?.parentId ?? null;
const seen = new Set<string>();
@ -119,7 +111,7 @@ function ancestorsOf(nodeId: string, byId: Map<string, NotionPickerNode>): strin
return ancestors;
}
function descendantsOf(nodeId: string, byId: Map<string, NotionPickerNode>): string[] {
function descendantsOf(nodeId: string, byId: Map<string, TreePickerNode>): string[] {
const result: string[] = [];
const stack = [...(byId.get(nodeId)?.childIds ?? [])].reverse();
while (stack.length > 0) {
@ -152,18 +144,18 @@ function matchingIds(state: PickerState): Set<string> {
);
}
export function buildPickerTree(searchResults: NotionPickerPageInput[]): NotionPickerNode[] {
export function buildPickerTree(inputs: TreePickerNodeInput[]): TreePickerNode[] {
const nodes = new Map<string, MutableNode>();
for (const result of searchResults) {
const id = normalizePageId(result.id);
if (nodes.has(id)) {
for (const result of inputs) {
const id = result.id.trim();
if (id.length === 0 || nodes.has(id)) {
continue;
}
nodes.set(id, {
id,
title: titleValue(result.title),
archived: result.archived === true,
parentId: result.parentId ? normalizePageId(result.parentId) : null,
parentId: result.parentId ? result.parentId.trim() : null,
childIds: [],
});
}
@ -202,7 +194,7 @@ export function buildPickerTree(searchResults: NotionPickerPageInput[]): NotionP
[...nodes.values()].filter((node) => node.parentId === null).map((node) => node.id),
nodes,
);
const tree: NotionPickerNode[] = [];
const tree: TreePickerNode[] = [];
function visit(nodeId: string, depth: number, pathPrefix: string[]): void {
const raw = nodes.get(nodeId);
@ -210,7 +202,7 @@ export function buildPickerTree(searchResults: NotionPickerPageInput[]): NotionP
return;
}
const path = [...pathPrefix, raw.title].join(' / ');
const node: NotionPickerNode = {
const node: TreePickerNode = {
id: raw.id,
title: raw.title,
archived: raw.archived,
@ -232,11 +224,11 @@ export function buildPickerTree(searchResults: NotionPickerPageInput[]): NotionP
return tree;
}
export function isAncestorChecked(nodeId: string, checked: Set<string>, byId: Map<string, NotionPickerNode>): boolean {
export function isAncestorChecked(nodeId: string, checked: Set<string>, byId: Map<string, TreePickerNode>): boolean {
return ancestorsOf(nodeId, byId).some((ancestorId) => checked.has(ancestorId));
}
function checkedAncestor(nodeId: string, state: PickerState): NotionPickerNode | null {
function checkedAncestor(nodeId: string, state: PickerState): TreePickerNode | null {
for (const ancestorId of ancestorsOf(nodeId, state.byId)) {
if (state.checked.has(ancestorId)) {
return state.byId.get(ancestorId) ?? null;
@ -247,7 +239,7 @@ function checkedAncestor(nodeId: string, state: PickerState): NotionPickerNode |
export function canToggle(nodeId: string, state: PickerState): { ok: true } | { ok: false; reason: string } {
if (!state.byId.has(nodeId)) {
return { ok: false, reason: 'Page not found' };
return { ok: false, reason: 'Node not found' };
}
const ancestor = checkedAncestor(nodeId, state);
if (ancestor) {
@ -276,7 +268,7 @@ export function toggleChecked(state: PickerState, nodeId: string, now = Date.now
return cloneState(state, { checked, transientHint: null });
}
export function flattenSelection(checked: Set<string>, byId: Map<string, NotionPickerNode>): string[] {
export function flattenSelection(checked: Set<string>, byId: Map<string, TreePickerNode>): string[] {
const result: string[] = [];
for (const node of byId.values()) {
if (checked.has(node.id) && !isAncestorChecked(node.id, checked, byId)) {
@ -402,16 +394,21 @@ export function moveCursor(state: PickerState, dir: 'up' | 'down' | 'left' | 'ri
}
export function buildInitialState(args: {
tree: NotionPickerNode[];
existingRootPageIds: string[];
currentCrawlMode?: 'all_accessible' | 'selected_roots';
tree: TreePickerNode[];
existingSelectedIds: string[];
requireConfirmOnSave?: boolean;
skipEmptyAction?: SkipEmptyAction;
staleWarning?: (staleCount: number) => string;
}): PickerState {
const byId = new Map(args.tree.map((node) => [node.id, node]));
const checked = new Set<string>();
let staleCount = 0;
for (const rawId of args.existingRootPageIds) {
const id = normalizePageId(rawId);
for (const rawId of args.existingSelectedIds) {
const id = rawId.trim();
if (id.length === 0) {
continue;
}
if (byId.has(id)) {
checked.add(id);
} else {
@ -427,6 +424,12 @@ export function buildInitialState(args: {
}
}
const preLoadWarnings: string[] = [];
if (staleCount > 0) {
const warning = args.staleWarning ? args.staleWarning(staleCount) : `${staleCount} stored selections no longer visible`;
preLoadWarnings.push(warning);
}
return {
tree: args.tree,
byId,
@ -435,16 +438,19 @@ export function buildInitialState(args: {
cursorId: args.tree[0]?.id ?? '',
search: { editing: false, query: '' },
pendingConfirm: null,
preLoadWarnings: staleCount > 0 ? [`${staleCount} stored root_page_ids no longer visible`] : [],
preLoadWarnings,
transientHint: null,
currentCrawlMode: args.currentCrawlMode ?? 'selected_roots',
requireConfirmOnSave: args.requireConfirmOnSave ?? false,
skipEmptyAction: args.skipEmptyAction ?? 'quit',
};
}
export function reducer(state: PickerState, cmd: PickerCommand, now = Date.now()): { next: PickerState; effect: PickerEffect } {
if (state.pendingConfirm) {
if (cmd === 'save-confirm') {
return { next: cloneState(state, { pendingConfirm: null }), effect: 'save' };
const effect: PickerEffect =
state.pendingConfirm === 'skip-empty' ? (state.skipEmptyAction === 'save-empty' ? 'save' : 'quit-without-save') : 'save';
return { next: cloneState(state, { pendingConfirm: null }), effect };
}
if (cmd === 'save-cancel') {
return { next: cloneState(state, { pendingConfirm: null }), effect: null };
@ -498,19 +504,13 @@ export function reducer(state: PickerState, cmd: PickerCommand, now = Date.now()
};
case 'save-request':
if (state.checked.size === 0) {
return {
next: cloneState(state, {
transientHint: transientHint('Select at least one page or press q to quit', now),
}),
effect: null,
};
return { next: cloneState(state, { pendingConfirm: 'skip-empty' }), effect: null };
}
if (state.currentCrawlMode === 'all_accessible') {
return { next: cloneState(state, { pendingConfirm: 'mode-switch' }), effect: null };
if (state.requireConfirmOnSave) {
return { next: cloneState(state, { pendingConfirm: 'save-confirm' }), effect: null };
}
return { next: state, effect: 'save' };
case 'save-confirm':
return { next: state, effect: 'save' };
case 'save-cancel':
return { next: state, effect: null };
case 'quit':

361
packages/cli/src/tree-picker-tui.test.tsx Normal file
View file

@ -0,0 +1,361 @@
/* @jsxImportSource react */
import { render as renderInkTest } from 'ink-testing-library';
import { type ReactNode } from 'react';
import { describe, expect, it, vi } from 'vitest';
import { buildInitialState, buildPickerTree, type TreePickerNodeInput } from './tree-picker-state.js';
import {
TreePickerApp,
renderTreePickerTui,
resolveTreePickerWidth,
sanitizeTreePickerTuiError,
treePickerCommandForInkInput,
windowItems,
windowOffset,
type TreePickerChrome,
type TreePickerInkInstance,
type TreePickerInkRenderOptions,
} from './tree-picker-tui.js';
const IDS = {
engineering: '11111111-1111-1111-1111-111111111111',
architecture: '22222222-2222-2222-2222-222222222222',
marketing: '33333333-3333-3333-3333-333333333333',
finance: '44444444-4444-4444-4444-444444444444',
ops: '55555555-5555-5555-5555-555555555555',
sales: '66666666-6666-6666-6666-666666666666',
support: '77777777-7777-7777-7777-777777777777',
product: '88888888-8888-8888-8888-888888888888',
design: '99999999-9999-9999-9999-999999999999',
};
function pages(): TreePickerNodeInput[] {
return [
{ id: IDS.engineering, title: 'Engineering Docs', archived: false, parentId: null },
{ id: IDS.architecture, title: 'Architecture', archived: false, parentId: IDS.engineering },
{ id: IDS.marketing, title: 'Marketing', archived: false, parentId: null },
];
}
function manyPages(): TreePickerNodeInput[] {
return [
{ id: IDS.engineering, title: 'Engineering Docs', archived: false, parentId: null },
{ id: IDS.architecture, title: 'Architecture', archived: false, parentId: IDS.engineering },
{ id: IDS.marketing, title: 'Marketing', archived: false, parentId: null },
{ id: IDS.finance, title: 'Finance', archived: false, parentId: null },
{ id: IDS.ops, title: 'Operations', archived: false, parentId: null },
{ id: IDS.sales, title: 'Sales', archived: false, parentId: null },
{ id: IDS.support, title: 'Support', archived: false, parentId: null },
{ id: IDS.product, title: 'Product', archived: false, parentId: null },
{ id: IDS.design, title: 'Design', archived: false, parentId: null },
];
}
function state(options: { requireConfirmOnSave?: boolean } = {}) {
return buildInitialState({
tree: buildPickerTree(pages()),
existingSelectedIds: [],
requireConfirmOnSave: options.requireConfirmOnSave ?? false,
});
}
function chrome(overrides: Partial<TreePickerChrome> = {}): TreePickerChrome {
return {
title: 'Select items',
subtitleLines: ['Source: Test'],
...overrides,
};
}
async function waitForInkInput(): Promise<void> {
await new Promise((resolve) => setTimeout(resolve, 10));
}
function fakeInkInstance(): TreePickerInkInstance {
return {
rerender: vi.fn(),
unmount: vi.fn(),
waitUntilExit: vi.fn(async () => undefined),
};
}
function normalizeFrameWrap(frame: string | undefined): string {
return frame?.replace(/\n/g, ' ').replace(/│ /g, '').replace(/ +/g, ' ') ?? '';
}
describe('treePickerCommandForInkInput', () => {
it('maps browse, search, and confirm input to reducer commands', () => {
expect(treePickerCommandForInkInput('', { downArrow: true }, state().search, null)).toBe('cursor-down');
expect(treePickerCommandForInkInput('', { upArrow: true }, state().search, null)).toBe('cursor-up');
expect(treePickerCommandForInkInput('', { rightArrow: true }, state().search, null)).toBe('cursor-right');
expect(treePickerCommandForInkInput('', { leftArrow: true }, state().search, null)).toBe('cursor-left');
expect(treePickerCommandForInkInput(' ', {}, state().search, null)).toBe('toggle-check');
expect(treePickerCommandForInkInput('/', {}, state().search, null)).toBe('search-start');
expect(treePickerCommandForInkInput('a', {}, state().search, null)).toBe('select-all-visible');
expect(treePickerCommandForInkInput('n', {}, state().search, null)).toBe('select-none');
expect(treePickerCommandForInkInput('', { return: true }, state().search, null)).toBe('save-request');
expect(treePickerCommandForInkInput('', { escape: true }, state().search, null)).toBe('quit');
expect(treePickerCommandForInkInput('c', { ctrl: true }, state().search, null)).toBe('quit');
expect(treePickerCommandForInkInput('s', {}, state().search, null)).toBeNull();
expect(treePickerCommandForInkInput('q', {}, state().search, null)).toBeNull();
expect(treePickerCommandForInkInput('x', {}, { editing: true, query: '' }, null)).toEqual({
type: 'search-input',
value: 'x',
});
expect(treePickerCommandForInkInput('', { backspace: true }, { editing: true, query: 'x' }, null)).toBe(
'search-backspace',
);
expect(treePickerCommandForInkInput('', { return: true }, { editing: true, query: 'x' }, null)).toBe(
'search-submit',
);
expect(treePickerCommandForInkInput('', { escape: true }, { editing: true, query: 'x' }, null)).toBe(
'search-cancel',
);
expect(treePickerCommandForInkInput('y', {}, state().search, 'save-confirm')).toBe('save-confirm');
expect(treePickerCommandForInkInput('', { return: true }, state().search, 'save-confirm')).toBe('save-confirm');
expect(treePickerCommandForInkInput('n', {}, state().search, 'save-confirm')).toBe('save-cancel');
});
});
describe('window helpers', () => {
it('centers the selected row and returns the visible slice', () => {
expect(windowOffset(20, 10, 5)).toBe(8);
expect(windowItems(['a', 'b', 'c', 'd', 'e'], 3, 3)).toEqual({ items: ['c', 'd', 'e'], offset: 2 });
});
it('clamps picker width to the design rule', () => {
expect(resolveTreePickerWidth(200)).toBe(120);
expect(resolveTreePickerWidth(100)).toBe(96);
expect(resolveTreePickerWidth(50)).toBe(60);
expect(resolveTreePickerWidth(undefined)).toBe(96);
});
});
describe('TreePickerApp', () => {
it('renders chrome title, subtitle, warnings, help, and row glyphs', () => {
const initialState = {
...state(),
preLoadWarnings: ['1 stale stored selections - they will be removed if you save'],
};
const { lastFrame } = renderInkTest(
<TreePickerApp
initialState={initialState}
chrome={chrome({
title: 'Select fancy widgets',
subtitleLines: ['Workspace: Design Workspace'],
warningLines: ['5000-page cap reached - some pages not shown'],
})}
terminalRows={24}
terminalWidth={100}
onExit={vi.fn()}
/>,
);
const frame = lastFrame() ?? '';
expect(frame).toContain('Select fancy widgets');
expect(frame).toContain('Workspace: Design Workspace');
expect(frame).toContain('5000-page cap reached - some pages not shown');
expect(frame).toContain('1 stale stored selections - they will be removed if you save');
expect(frame).toContain('◻ Engineering Docs ▸ (1)');
expect(frame).toContain('◻ Marketing');
expect(normalizeFrameWrap(frame)).toContain(
'Right Arrow to expand, Up/Down to move, Space to select or unselect, Slash to filter, Enter to confirm, Escape to go back, or Ctrl+C to exit.',
);
});
it('renders custom help text when supplied', () => {
const { lastFrame } = renderInkTest(
<TreePickerApp
initialState={state()}
chrome={chrome({ helpText: 'Bespoke instructions here.' })}
terminalRows={24}
terminalWidth={100}
onExit={vi.fn()}
/>,
);
expect(lastFrame() ?? '').toContain('Bespoke instructions here.');
});
it('renders checked parents and locked descendants with locked glyphs', () => {
const initialState = {
...state(),
checked: new Set([IDS.engineering]),
expanded: new Set([IDS.engineering]),
};
const { lastFrame } = renderInkTest(
<TreePickerApp
initialState={initialState}
chrome={chrome()}
terminalRows={24}
terminalWidth={100}
onExit={vi.fn()}
/>,
);
const frame = lastFrame() ?? '';
expect(frame).toContain('◼ Engineering Docs ▾');
expect(frame).toContain(' ◼ Architecture');
});
it('supports keyboard selection, confirm-on-save, and save callback', async () => {
const onExit = vi.fn();
const { stdin, lastFrame } = renderInkTest(
<TreePickerApp
initialState={state({ requireConfirmOnSave: true })}
chrome={chrome({
confirmSaveMessage: (current) =>
`Confirm: ${current.checked.size} item${current.checked.size === 1 ? '' : 's'}? Press Enter or Escape.`,
})}
terminalRows={24}
terminalWidth={100}
onExit={onExit}
/>,
);
stdin.write(' ');
await waitForInkInput();
expect(lastFrame()).toContain('◼ Engineering Docs');
stdin.write('\r');
await waitForInkInput();
expect(normalizeFrameWrap(lastFrame())).toContain('Confirm: 1 item? Press Enter or Escape.');
stdin.write('y');
await waitForInkInput();
expect(onExit).toHaveBeenCalledWith({ kind: 'save', selectedIds: [IDS.engineering] });
});
it('uses the chrome-supplied skip-empty message and quits on confirm', async () => {
const onExit = vi.fn();
const { stdin, lastFrame } = renderInkTest(
<TreePickerApp
initialState={state()}
chrome={chrome({ skipEmptyMessage: 'No selections. Skip or back?' })}
terminalRows={24}
terminalWidth={100}
onExit={onExit}
/>,
);
stdin.write('\r');
await waitForInkInput();
expect(normalizeFrameWrap(lastFrame())).toContain('No selections. Skip or back?');
expect(onExit).not.toHaveBeenCalled();
stdin.write('n');
await waitForInkInput();
expect(lastFrame()).not.toContain('No selections. Skip or back?');
expect(onExit).not.toHaveBeenCalled();
stdin.write('\r');
await waitForInkInput();
expect(lastFrame()).toContain('No selections. Skip or back?');
stdin.write('\r');
await waitForInkInput();
expect(onExit).toHaveBeenCalledWith({ kind: 'quit' });
});
it('renders row-window overflow indicators when the visible list is clipped', async () => {
const onExit = vi.fn();
const initialState = buildInitialState({
tree: buildPickerTree(manyPages()),
existingSelectedIds: [],
});
initialState.expanded = new Set([IDS.engineering]);
const { stdin, lastFrame } = renderInkTest(
<TreePickerApp
initialState={initialState}
chrome={chrome()}
terminalRows={13}
terminalWidth={100}
onExit={onExit}
/>,
);
expect(lastFrame()).toContain('↓ 4 more');
stdin.write('');
stdin.write('');
stdin.write('');
stdin.write('');
await waitForInkInput();
const frame = lastFrame() ?? '';
expect(frame).toContain('↑ ');
expect(frame).toContain('↓ ');
expect(onExit).not.toHaveBeenCalled();
});
it('quits without saving on Ctrl+C', async () => {
const onExit = vi.fn();
const { stdin } = renderInkTest(
<TreePickerApp
initialState={state()}
chrome={chrome()}
terminalRows={24}
terminalWidth={100}
onExit={onExit}
/>,
);
stdin.write('');
await waitForInkInput();
expect(onExit).toHaveBeenCalledWith({ kind: 'quit' });
});
});
describe('renderTreePickerTui', () => {
it('returns the app result from the Ink runtime', async () => {
const io = {
stdin: { isTTY: true, setRawMode: vi.fn() },
stdout: { isTTY: true, columns: 100, rows: 24, write: vi.fn() },
stderr: { write: vi.fn() },
};
const renderInk = vi.fn((_tree: ReactNode, _options: TreePickerInkRenderOptions) => fakeInkInstance());
await expect(
renderTreePickerTui(
{ initialState: state(), chrome: chrome() },
io,
{ renderInk },
),
).resolves.toEqual({ kind: 'quit' });
expect(renderInk).toHaveBeenCalledOnce();
});
it('sanitizes render errors and uses the supplied scripted-mode hint', async () => {
expect(sanitizeTreePickerTuiError(new Error('token=secret https://api.example.com/v1/search'))).toBe(
'[redacted] [redacted-url]',
);
});
it('falls back to quit with the scripted-mode hint when Ink cannot initialize', async () => {
let stderr = '';
const io = {
stdin: { isTTY: false, setRawMode: vi.fn() },
stdout: { isTTY: false, columns: 100, rows: 24, write: vi.fn() },
stderr: {
write(chunk: string) {
stderr += chunk;
},
},
};
await expect(
renderTreePickerTui(
{ initialState: state(), chrome: chrome() },
io,
{
renderInk: vi.fn(() => {
throw new Error('token=secret');
}),
scriptedModeHint: 'Use --no-input --foo bar for scripted mode.',
},
),
).resolves.toEqual({ kind: 'quit' });
expect(stderr).toContain('Use --no-input --foo bar for scripted mode.');
expect(stderr).not.toContain('secret');
});
});

221
packages/cli/src/notion-page-picker-tui.tsx → packages/cli/src/tree-picker-tui.tsx
View file

@ -9,13 +9,14 @@ import {
visibleNodeIds,
type PickerCommand,
type PickerState,
} from './notion-page-picker-tree.js';
} from './tree-picker-state.js';
import type { KtxCliIo } from './cli-runtime.js';
const COLOR_THEME = {
text: 'white',
muted: 'gray',
active: 'cyan',
selected: 'green',
warning: 'yellow',
} as const;
@ -23,12 +24,19 @@ const NO_COLOR_THEME = {
text: 'white',
muted: 'white',
active: 'white',
selected: 'white',
warning: 'white',
} as const;
type NotionPickerTheme = Record<keyof typeof COLOR_THEME, string>;
type TreePickerTheme = Record<keyof typeof COLOR_THEME, string>;
export interface NotionPickerTuiIo extends KtxCliIo {
const DEFAULT_TREE_PICKER_HELP_TEXT =
'Right Arrow to expand, Up/Down to move, Space to select or unselect, Slash to filter, Enter to confirm, Escape to go back, or Ctrl+C to exit.';
const DEFAULT_SKIP_EMPTY_MESSAGE =
'Nothing selected. Skip this step? Press Enter to skip or Escape to go back.';
export interface TreePickerTuiIo extends KtxCliIo {
stdin?: { isTTY?: boolean; setRawMode?(value: boolean): void };
stdout: KtxCliIo['stdout'] & { isTTY?: boolean; columns?: number; rows?: number };
}
@ -45,58 +53,54 @@ interface InkKey {
delete?: boolean;
}
export type PickerRenderResult = { kind: 'save'; rootPageIds: string[] } | { kind: 'quit' };
export type TreePickerResult = { kind: 'save'; selectedIds: string[] } | { kind: 'quit' };
export interface PickerRenderInput {
initialState: PickerState;
connectionId: string;
workspaceLabel: string;
cappedAtCount: number | null;
currentCrawlMode: 'all_accessible' | 'selected_roots';
export interface TreePickerChrome {
title: string;
helpText?: string;
subtitleLines?: readonly string[];
warningLines?: readonly string[];
confirmSaveMessage?: (state: PickerState) => string;
skipEmptyMessage?: string;
}
interface NotionPickerAppProps extends PickerRenderInput {
export interface TreePickerRenderInput {
initialState: PickerState;
chrome: TreePickerChrome;
}
interface TreePickerAppProps extends TreePickerRenderInput {
terminalRows?: number;
terminalWidth?: number;
env?: NodeJS.ProcessEnv;
onExit(result: PickerRenderResult): void;
onExit(result: TreePickerResult): void;
}
export interface NotionPickerInkInstance {
export interface TreePickerInkInstance {
rerender(tree: ReactNode): void;
unmount(): void;
waitUntilExit(): Promise<void>;
}
export interface NotionPickerInkRenderOptions {
stdin?: NotionPickerTuiIo['stdin'];
stdout: NotionPickerTuiIo['stdout'];
stderr: NotionPickerTuiIo['stderr'];
export interface TreePickerInkRenderOptions {
stdin?: TreePickerTuiIo['stdin'];
stdout: TreePickerTuiIo['stdout'];
stderr: TreePickerTuiIo['stderr'];
exitOnCtrlC: boolean;
patchConsole: boolean;
maxFps: number;
alternateScreen: boolean;
}
function resolveTheme(env: NodeJS.ProcessEnv = process.env): NotionPickerTheme {
function resolveTheme(env: NodeJS.ProcessEnv = process.env): TreePickerTheme {
return env.NO_COLOR || env.TERM === 'dumb' ? NO_COLOR_THEME : COLOR_THEME;
}
export function resolveNotionPickerWidth(columns: number | undefined): number {
export function resolveTreePickerWidth(columns: number | undefined): number {
const resolvedColumns = columns ?? 100;
return Math.max(60, Math.min(120, resolvedColumns - 4));
}
function staleWarningText(warning: string): string {
return warning.includes('stored root_page_ids no longer visible')
? `${warning} - they will be removed if you save`
: warning;
}
function selectedPageCountText(count: number): string {
return `${count} selected ${count === 1 ? 'page' : 'pages'}`;
}
function rowMatchesSearch(state: PickerState, nodeId: string): boolean {
const query = state.search.query.trim().toLocaleLowerCase();
if (!query) {
@ -109,7 +113,7 @@ function rowMatchesSearch(state: PickerState, nodeId: string): boolean {
return node.title.toLocaleLowerCase().includes(query) || node.path.toLocaleLowerCase().includes(query);
}
export function sanitizeNotionPickerTuiError(error: unknown): string {
export function sanitizeTreePickerTuiError(error: unknown): string {
const message = error instanceof Error ? error.message : String(error);
return message
.replace(/[a-z][a-z0-9+.-]*:\/\/[^\s]+/gi, '[redacted-url]')
@ -132,7 +136,7 @@ function truncateText(value: string, width: number): string {
return `${value.slice(0, width - 3)}...`;
}
export function notionPickerCommandForInkInput(
export function treePickerCommandForInkInput(
input: string,
key: InkKey,
search: PickerState['search'],
@ -150,7 +154,7 @@ export function notionPickerCommandForInkInput(
if (key.backspace || key.delete) return 'search-backspace';
if (key.downArrow) return 'cursor-down';
if (key.upArrow) return 'cursor-up';
if (input.length === 1 && input >= ' ' && input !== '\u007f') return { type: 'search-input', value: input };
if (input.length === 1 && input >= ' ' && input !== '') return { type: 'search-input', value: input };
return null;
}
if (key.ctrl === true && input === 'c') return 'quit';
@ -158,54 +162,64 @@ export function notionPickerCommandForInkInput(
if (key.downArrow) return 'cursor-down';
if (key.leftArrow) return 'cursor-left';
if (key.rightArrow) return 'cursor-right';
if (key.return) return 'expand';
if (key.return) return 'save-request';
if (input === ' ') return 'toggle-check';
if (input === '/') return 'search-start';
if (input === 'a') return 'select-all-visible';
if (input === 'n') return 'select-none';
if (input === 's') return 'save-request';
if (input === 'q' || key.escape) return 'quit';
if (key.escape) return 'quit';
return null;
}
function PickerRow(props: { state: PickerState; nodeId: string; width: number; theme: NotionPickerTheme }): ReactNode {
function PickerRow(props: { state: PickerState; nodeId: string; width: number; theme: TreePickerTheme }): ReactNode {
const node = props.state.byId.get(props.nodeId);
if (!node) return null;
const focused = props.state.cursorId === node.id;
const locked = isAncestorChecked(node.id, props.state.checked, props.state.byId);
const checked = props.state.checked.has(node.id);
const glyph = locked ? '[~]' : checked ? '[×]' : '[ ]';
const children =
const isSelected = checked || locked;
const glyph = isSelected ? '◼' : '◻';
const glyphColor = checked || locked ? props.theme.selected : props.theme.muted;
const childAffordance =
node.childIds.length > 0 ? (props.state.expanded.has(node.id) ? ' ▾' : ` ▸ (${node.childIds.length})`) : '';
const prefix = `${focused ? '▸' : ' '} ${glyph} ${' '.repeat(node.depth * 2)}`;
const color = focused ? props.theme.active : locked || node.archived ? props.theme.muted : props.theme.text;
const title = truncateText(`${node.title}${children}`, Math.max(10, props.width - prefix.length));
const indent = ' '.repeat(node.depth * 2);
const titleColor = focused ? props.theme.active : props.theme.text;
const inverse = rowMatchesSearch(props.state, node.id);
const prefixWidth = indent.length + 2 + childAffordance.length;
const title = truncateText(node.title, Math.max(10, props.width - prefixWidth));
return (
<Text color={color} strikethrough={node.archived}>
{prefix}
<Text inverse={inverse}>{title}</Text>
<Text>
<Text color={glyphColor}>
{indent}
{glyph}
</Text>
<Text color={titleColor} strikethrough={node.archived} bold={focused}>
{' '}
<Text inverse={inverse}>{title}</Text>
</Text>
{childAffordance.length > 0 ? <Text color={props.theme.muted}>{childAffordance}</Text> : null}
</Text>
);
}
export function NotionPickerApp(props: NotionPickerAppProps): ReactNode {
export function TreePickerApp(props: TreePickerAppProps): ReactNode {
const app = useApp();
const [state, setState] = useState(props.initialState);
const stateRef = useRef(state);
const theme = useMemo(() => resolveTheme(props.env), [props.env]);
const visibleIds = visibleNodeIds(state);
const selectedIndex = Math.max(0, visibleIds.indexOf(state.cursorId));
const reservedRows = state.pendingConfirm === 'mode-switch' ? 9 : 8;
const visibleRows = Math.max(5, Math.min(20, (props.terminalRows ?? 24) - reservedRows));
const reservedRows = state.pendingConfirm === 'save-confirm' ? 10 : 9;
const visibleRows = Math.max(5, Math.min(12, (props.terminalRows ?? 24) - reservedRows));
const rows = windowItems(visibleIds, selectedIndex, visibleRows);
const hiddenAbove = rows.offset;
const hiddenBelow = Math.max(0, visibleIds.length - rows.offset - rows.items.length);
const searchMatchCount = filterTree(state).visibleIds.size;
const width = resolveNotionPickerWidth(props.terminalWidth);
const width = resolveTreePickerWidth(props.terminalWidth);
const showSearch = state.search.editing || state.search.query.trim().length > 0;
const selectedCount = flattenSelection(state.checked, state.byId).length;
const helpText = props.chrome.helpText ?? DEFAULT_TREE_PICKER_HELP_TEXT;
const skipEmptyMessage = props.chrome.skipEmptyMessage ?? DEFAULT_SKIP_EMPTY_MESSAGE;
stateRef.current = state;
@ -234,7 +248,7 @@ export function NotionPickerApp(props: NotionPickerAppProps): ReactNode {
}, [state.transientHint?.expiresAt]);
useInput((input, key) => {
const command = notionPickerCommandForInkInput(input, key, stateRef.current.search, stateRef.current.pendingConfirm);
const command = treePickerCommandForInkInput(input, key, stateRef.current.search, stateRef.current.pendingConfirm);
if (!command) {
return;
}
@ -242,7 +256,7 @@ export function NotionPickerApp(props: NotionPickerAppProps): ReactNode {
stateRef.current = next;
setState(next);
if (effect === 'save') {
props.onExit({ kind: 'save', rootPageIds: flattenSelection(next.checked, next.byId) });
props.onExit({ kind: 'save', selectedIds: flattenSelection(next.checked, next.byId) });
app.exit();
return;
}
@ -254,39 +268,68 @@ export function NotionPickerApp(props: NotionPickerAppProps): ReactNode {
return (
<Box flexDirection="column">
<Text color={theme.active}>Notion pages visible to integration "{props.workspaceLabel}"</Text>
{props.cappedAtCount ? <Text color={theme.warning}>{props.cappedAtCount}-page cap reached - some pages not shown</Text> : null}
{state.preLoadWarnings.map((warning) => (
<Text key={warning} color={theme.warning}>
{staleWarningText(warning)}
</Text>
))}
{showSearch ? (
<Text color={theme.muted}>
/ {state.search.query}
{state.search.editing ? '█' : ''} ({searchMatchCount} matches)
</Text>
) : null}
<Box flexDirection="column">
<Text>
<Text color={theme.active}>◆</Text>
<Text bold> {props.chrome.title}</Text>
</Text>
<Box
flexDirection="column"
borderStyle="single"
borderTop={false}
borderRight={false}
borderBottom={false}
borderColor={theme.active}
paddingLeft={1}
>
<Text color={theme.muted}>{helpText}</Text>
<Text> </Text>
{(props.chrome.subtitleLines ?? []).map((line, idx) => (
<Text key={`subtitle-${idx}`} color={theme.muted}>
{line}
</Text>
))}
{(props.chrome.warningLines ?? []).map((line, idx) => (
<Text key={`chromewarn-${idx}`} color={theme.warning}>
{line}
</Text>
))}
{state.preLoadWarnings.map((warning) => (
<Text key={warning} color={theme.warning}>
{warning}
</Text>
))}
{showSearch ? (
<Text>
<Text color={theme.muted}>/ </Text>
<Text>
{state.search.query}
{state.search.editing ? '█' : ''}
</Text>
<Text color={theme.muted}> ({searchMatchCount} matches)</Text>
</Text>
) : null}
<Text> </Text>
{hiddenAbove > 0 ? <Text color={theme.muted}>↑ {hiddenAbove} more</Text> : null}
{rows.items.map((nodeId) => (
<PickerRow key={nodeId} state={state} nodeId={nodeId} width={width} theme={theme} />
))}
{hiddenBelow > 0 ? <Text color={theme.muted}>↓ {hiddenBelow} more</Text> : null}
{state.pendingConfirm === 'save-confirm' ? (
<Text color={theme.warning}>
{props.chrome.confirmSaveMessage
? props.chrome.confirmSaveMessage(state)
: 'Confirm save? Press Enter to confirm or Escape to go back.'}
</Text>
) : null}
{state.pendingConfirm === 'skip-empty' ? <Text color={theme.warning}>{skipEmptyMessage}</Text> : null}
{state.transientHint ? <Text color={theme.warning}>{state.transientHint.text}</Text> : null}
</Box>
{state.pendingConfirm === 'mode-switch' ? (
<Text color={theme.warning}>
Save will switch crawl_mode all_accessible -&gt; selected_roots and limit ingest to{' '}
{selectedPageCountText(selectedCount)}. [y] confirm [esc] back
</Text>
) : null}
{state.transientHint ? <Text color={theme.warning}>{state.transientHint.text}</Text> : null}
<Text color={theme.muted}>space toggle · enter expand · / search · a all · n none · s save &amp; exit · q quit</Text>
<Text color={theme.active}>└</Text>
</Box>
);
}
function renderInk(tree: ReactNode, options: NotionPickerInkRenderOptions): NotionPickerInkInstance {
function renderInk(tree: ReactNode, options: TreePickerInkRenderOptions): TreePickerInkInstance {
return renderInkRuntime(tree, {
stdin: options.stdin as NodeJS.ReadStream | undefined,
stdout: options.stdout as NodeJS.WriteStream,
@ -295,19 +338,24 @@ function renderInk(tree: ReactNode, options: NotionPickerInkRenderOptions): Noti
patchConsole: options.patchConsole,
maxFps: options.maxFps,
alternateScreen: options.alternateScreen,
}) as NotionPickerInkInstance;
}) as TreePickerInkInstance;
}
export async function renderNotionPickerTui(
input: PickerRenderInput,
io: NotionPickerTuiIo,
options: { renderInk?: (tree: ReactNode, options: NotionPickerInkRenderOptions) => NotionPickerInkInstance } = {},
): Promise<PickerRenderResult> {
let result: PickerRenderResult = { kind: 'quit' };
let instance: NotionPickerInkInstance | null = null;
export interface RenderTreePickerOptions {
renderInk?: (tree: ReactNode, options: TreePickerInkRenderOptions) => TreePickerInkInstance;
scriptedModeHint?: string;
}
export async function renderTreePickerTui(
input: TreePickerRenderInput,
io: TreePickerTuiIo,
options: RenderTreePickerOptions = {},
): Promise<TreePickerResult> {
let result: TreePickerResult = { kind: 'quit' };
let instance: TreePickerInkInstance | null = null;
try {
instance = (options.renderInk ?? renderInk)(
<NotionPickerApp
<TreePickerApp
{...input}
terminalRows={(io.stdout as { rows?: number }).rows ?? process.stdout.rows ?? 24}
terminalWidth={io.stdout.columns ?? process.stdout.columns}
@ -323,16 +371,15 @@ export async function renderNotionPickerTui(
exitOnCtrlC: false,
patchConsole: false,
maxFps: 30,
alternateScreen: true,
alternateScreen: false,
},
);
await instance.waitUntilExit();
instance.unmount();
return result;
} catch (error) {
io.stderr.write(
`Notion picker requires a TTY. Use --no-input --notion-root-page-id <UUID> for scripted mode. ${sanitizeNotionPickerTuiError(error)}\n`,
);
const hint = options.scriptedModeHint ?? 'Picker requires a TTY.';
io.stderr.write(`${hint} ${sanitizeTreePickerTuiError(error)}\n`);
return { kind: 'quit' };
}
}

2
packages/context/src/ingest/adapters/historic-sql/redaction.test.ts
View file

@ -9,7 +9,7 @@ describe('historic-SQL redaction', () => {
]);
const sql =
"select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'";
"select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; // pragma: allowlist secret
expect(redactHistoricSqlText(sql, redactors)).toBe(
"select * from public.api_events where api_key = '[REDACTED]' and note = '[REDACTED]'",

2
packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts
View file

@ -169,7 +169,7 @@ describe('stageHistoricSqlAggregatedSnapshot', () => {
it('redacts configured SQL substrings in staged artifacts while analyzing original SQL', async () => {
const stagedDir = await tempDir();
const originalSql =
"select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'";
"select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; // pragma: allowlist secret
const reader: HistoricSqlReader = {
async probe() {
return { warnings: [], info: [] };

2
packages/context/src/ingest/adapters/metabase/client.test.ts
View file

@ -92,7 +92,7 @@ describe('MetabaseClient retry exhaustion', () => {
.mockResolvedValueOnce(new Response(JSON.stringify([]), { status: 200 }));
const client = new MetabaseClient(
{ apiUrl: 'https://metabase.example.test', apiKey: 'key' },
{ apiUrl: 'https://metabase.example.test', apiKey: 'key' }, // pragma: allowlist secret
{
...DEFAULT_METABASE_CLIENT_CONFIG,
baseDelayMs: 0,

2
packages/context/src/ingest/adapters/metabase/local-metabase.adapter.test.ts
View file

@ -39,7 +39,7 @@ describe('metabaseRuntimeConfigFromLocalConnection', () => {
const connection: KtxProjectConnectionConfig = {
driver: 'metabase',
api_url: 'https://metabase.example.com',
api_key_ref: `file:${keyPath}`,
api_key_ref: `file:${keyPath}`, // pragma: allowlist secret
};
expect(metabaseRuntimeConfigFromLocalConnection('prod-metabase', connection)).toEqual({

10
packages/llm/src/embedding-health.test.ts
View file

@ -17,13 +17,13 @@ describe('KTX embedding health check', () => {
backend: 'openai',
model: 'text-embedding-3-small',
dimensions: 3,
openai: { apiKey: 'sk-openai-test' },
openai: { apiKey: 'sk-openai-test' }, // pragma: allowlist secret
},
{ deps: { createOpenAIClient } },
),
).resolves.toEqual({ ok: true });
expect(createOpenAIClient).toHaveBeenCalledWith({ apiKey: 'sk-openai-test', baseURL: undefined });
expect(createOpenAIClient).toHaveBeenCalledWith({ apiKey: 'sk-openai-test', baseURL: undefined }); // pragma: allowlist secret
});
it('returns failed when the provider returns the wrong dimensions', async () => {
@ -41,7 +41,7 @@ describe('KTX embedding health check', () => {
backend: 'openai',
model: 'text-embedding-3-small',
dimensions: 3,
openai: { apiKey: 'sk-openai-test' },
openai: { apiKey: 'sk-openai-test' }, // pragma: allowlist secret
},
{ deps: { createOpenAIClient } },
),
@ -66,7 +66,7 @@ describe('KTX embedding health check', () => {
backend: 'openai',
model: 'text-embedding-3-small',
dimensions: 3,
openai: { apiKey: 'sk-openai-secret' },
openai: { apiKey: 'sk-openai-secret' }, // pragma: allowlist secret
},
{ deps: { createOpenAIClient } },
),
@ -94,7 +94,7 @@ describe('KTX embedding health check', () => {
backend: 'openai',
model: 'text-embedding-3-small',
dimensions: 3,
openai: { apiKey: 'sk-openai-test' },
openai: { apiKey: 'sk-openai-test' }, // pragma: allowlist secret
},
{ timeoutMs: 1, deps: { createOpenAIClient } },
),

6
packages/llm/src/model-health.test.ts
View file

@ -14,7 +14,7 @@ describe('KTX LLM health check', () => {
runKtxLlmHealthCheck(
{
backend: 'anthropic',
anthropic: { apiKey: 'sk-ant-test' },
anthropic: { apiKey: 'sk-ant-test' }, // pragma: allowlist secret
modelSlots: { default: 'claude-sonnet-4-6' },
},
{ deps: { createAnthropic, generateText, devtoolsEnabled: true, wrapLanguageModel } },
@ -23,7 +23,7 @@ describe('KTX LLM health check', () => {
expect(createAnthropic).toHaveBeenCalledWith(
expect.objectContaining({
apiKey: 'sk-ant-test',
apiKey: 'sk-ant-test', // pragma: allowlist secret
}),
);
expect(generateText).toHaveBeenCalledWith(
@ -46,7 +46,7 @@ describe('KTX LLM health check', () => {
runKtxLlmHealthCheck(
{
backend: 'anthropic',
anthropic: { apiKey: 'sk-ant-secret' },
anthropic: { apiKey: 'sk-ant-secret' }, // pragma: allowlist secret
modelSlots: { default: 'claude-sonnet-4-6' },
},
{

1
pyproject.toml
View file

@ -13,6 +13,7 @@ Issues = "https://github.com/kaelio/ktx/issues"
[dependency-groups]
dev = [
"pre-commit>=4.6.0",
"pytest>=9.0.2",
"ruff>=0.8.4",
]

4
python/ktx-daemon/src/ktx_daemon/sql_analysis.py
View file

@ -130,7 +130,9 @@ def _analyze_one(
)
def _analyze_payload(payload: tuple[str, str, str]) -> tuple[str, AnalyzeSqlBatchResult]:
def _analyze_payload(
payload: tuple[str, str, str],
) -> tuple[str, AnalyzeSqlBatchResult]:
item_id, sql, dialect = payload
return _analyze_one(item_id, sql, dialect)

6
scripts/public-benchmark-manifest.json
View file

@ -4,7 +4,7 @@
"id": "chinook_with_declared_metadata",
"displayName": "Chinook (SQLite, declared metadata)",
"url": "https://raw.githubusercontent.com/lerocha/chinook-database/master/ChinookDatabase/DataSources/Chinook_Sqlite.sqlite",
"sha256": "7651ba378ac2fcd0dfc3c66fb101f7a7eed3ba39a612ec642b96e20702061f15",
"sha256": "7651ba378ac2fcd0dfc3c66fb101f7a7eed3ba39a612ec642b96e20702061f15", "_allowlist": "// pragma: allowlist secret",
"license": "MIT",
"source": "https://github.com/lerocha/chinook-database"
},
@ -12,7 +12,7 @@
"id": "northwind_with_declared_metadata",
"displayName": "Northwind (SQLite, declared metadata)",
"url": "https://github.com/jpwhite3/northwind-SQLite3/raw/main/dist/northwind.db",
"sha256": "2f4f5c68dfcd33ba27373eae48c7a4869800c68095ee0f9f0da494f83382a877",
"sha256": "2f4f5c68dfcd33ba27373eae48c7a4869800c68095ee0f9f0da494f83382a877", "_allowlist": "// pragma: allowlist secret",
"license": "MIT",
"source": "https://github.com/jpwhite3/northwind-SQLite3"
},
@ -20,7 +20,7 @@
"id": "sakila_with_declared_metadata",
"displayName": "Sakila (SQLite, declared metadata)",
"url": "https://raw.githubusercontent.com/bradleygrant/sakila-sqlite3/master/sakila_master.db",
"sha256": "88c91a4a1a6b61f9d3f35904c0a173c887b25e73f20c3c2fdb073818c06f4268",
"sha256": "88c91a4a1a6b61f9d3f35904c0a173c887b25e73f20c3c2fdb073818c06f4268", "_allowlist": "// pragma: allowlist secret",
"license": "BSD-2-Clause",
"source": "https://github.com/bradleygrant/sakila-sqlite3"
},

7
scripts/standalone-ci-workflow.test.mjs
View file

@ -20,6 +20,8 @@ describe('standalone KTX CI workflow', () => {
assertIncludesAll(workflow, [
'permissions:',
'contents: read',
'pre-commit-checks:',
'name: Pre-commit checks',
'typescript-checks:',
'name: TypeScript checks',
'slow-context-tests:',
@ -33,7 +35,7 @@ describe('standalone KTX CI workflow', () => {
'artifact-checks:',
'name: Artifact checks',
'actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd',
'pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0',
'pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093',
'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e',
'node-version: "24"',
'cache-dependency-path: "pnpm-lock.yaml"',
@ -46,7 +48,10 @@ describe('standalone KTX CI workflow', () => {
'actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405',
'python-version: "3.13"',
'astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b',
'version: "0.11.11"',
'cache-dependency-glob: "uv.lock"',
'uv sync --all-packages --all-groups',
'uv run pre-commit run --all-files',
'uv sync --all-packages',
'uv run pytest',
'pnpm run artifacts:check',

2
uv.lock generated
View file

@ -546,6 +546,7 @@ source = { virtual = "." }
[package.dev-dependencies]
dev = [
{ name = "pre-commit" },
{ name = "pytest" },
{ name = "ruff" },
]
@ -554,6 +555,7 @@ dev = [
[package.metadata.requires-dev]
dev = [
{ name = "pre-commit", specifier = ">=4.6.0" },
{ name = "pytest", specifier = ">=9.0.2" },
{ name = "ruff", specifier = ">=0.8.4" },
]