diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3da14c7b..ff0b7843 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,6 +15,46 @@ concurrency: cancel-in-progress: true jobs: + pre-commit-checks: + name: Pre-commit checks + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Setup pnpm + uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 + with: + run_install: false + + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: "24" + cache: "pnpm" + cache-dependency-path: "pnpm-lock.yaml" + + - name: Install TypeScript dependencies + run: pnpm install --frozen-lockfile + + - name: Setup Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.13" + + - name: Setup uv + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + with: + version: "0.11.11" + enable-cache: true + cache-dependency-glob: "uv.lock" + + - name: Install Python dependencies + run: uv sync --all-packages --all-groups + + - name: Run pre-commit hooks + run: uv run pre-commit run --all-files + typescript-checks: name: TypeScript checks runs-on: ubuntu-latest @@ -23,7 +63,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup pnpm - uses: pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0 # v6.0.7 + uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 with: run_install: false @@ -51,7 +91,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup pnpm - uses: pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0 # v6.0.7 + uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 with: run_install: false @@ -79,7 +119,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup pnpm - uses: pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0 # v6.0.7 + uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 with: run_install: false @@ -107,7 +147,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup pnpm - uses: pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0 # v6.0.7 + uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 with: run_install: false @@ -156,7 +196,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup pnpm - uses: pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0 # v6.0.7 + uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 with: run_install: false diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2a8f696e..36eaf49c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,7 +24,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup pnpm - uses: pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0 # v6.0.7 + uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 with: run_install: false diff --git a/LICENSE b/LICENSE index 57bc88a1..261eeb9e 100644 --- a/LICENSE +++ b/LICENSE @@ -199,4 +199,3 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - diff --git a/docs-site/content/docs/cli-reference/ktx-setup.mdx b/docs-site/content/docs/cli-reference/ktx-setup.mdx index f490988a..59fbe666 100644 --- a/docs-site/content/docs/cli-reference/ktx-setup.mdx +++ b/docs-site/content/docs/cli-reference/ktx-setup.mdx @@ -18,8 +18,6 @@ ktx setup [options] | Flag | Description | Default | |------|-------------|---------| | `--project-dir ` | KTX project directory | `KTX_PROJECT_DIR`, nearest `ktx.yaml`, or cwd | -| `--new` | Create a new KTX project before setup | `false` | -| `--existing` | Use an existing KTX project | `false` | | `--yes` | Accept safe defaults in non-interactive setup | `false` | | `--no-input` | Disable interactive terminal input | — | @@ -29,76 +27,11 @@ ktx setup [options] |------|-------------|---------| | `--agents` | Install agent integration only | `false` | | `--target ` | Agent target (`claude-code`, `codex`, `cursor`, `opencode`, `universal`) | — | -| `--agent-scope ` | Agent install scope (`project` or `global`) | `project` | -| `--project` | Install agent integration into the project scope | `false` | | `--global` | Install agent integration into the global target scope (Claude Code and Codex only) | `false` | -| `--skip-agents` | Leave agent integration incomplete for now | `false` | -### LLM Configuration - -| Flag | Description | Default | -|------|-------------|---------| -| `--anthropic-api-key-env ` | Environment variable containing the Anthropic API key | — | -| `--anthropic-api-key-file ` | File containing the Anthropic API key | — | -| `--anthropic-model ` | Anthropic model ID to validate and save | — | -| `--skip-llm` | Leave LLM setup incomplete for now | `false` | - -### Embedding Configuration - -| Flag | Description | Default | -|------|-------------|---------| -| `--embedding-backend ` | Embedding backend (`openai` or `sentence-transformers`) | — | -| `--embedding-api-key-env ` | Environment variable containing the embedding provider API key | — | -| `--embedding-api-key-file ` | File containing the embedding provider API key | — | -| `--skip-embeddings` | Leave embedding setup incomplete for now | `false` | - -### Database Configuration - -| Flag | Description | Default | -|------|-------------|---------| -| `--database ` | Database driver to configure; repeatable (`sqlite`, `postgres`, `mysql`, `clickhouse`, `sqlserver`, `bigquery`, `snowflake`) | — | -| `--database-connection-id ` | Existing or new connection id; repeatable | — | -| `--new-database-connection-id ` | Connection id for one new database connection | — | -| `--database-url ` | URL, `env:NAME`, or `file:/path` for one new URL-style database connection | — | -| `--database-schema ` | Database schema to include; repeatable | — | -| `--skip-databases` | Leave database setup incomplete | `false` | - -### Historic SQL - -| Flag | Description | Default | -|------|-------------|---------| -| `--enable-historic-sql` | Enable Historic SQL when the selected database supports it | `false` | -| `--disable-historic-sql` | Disable Historic SQL for the selected database | `false` | -| `--historic-sql-window-days ` | Historic SQL query-history window in days | — | -| `--historic-sql-min-executions ` | Minimum executions for a Historic SQL template | — | -| `--historic-sql-min-calls ` | Alias for `--historic-sql-min-executions` for one release | — | -| `--historic-sql-service-account-pattern ` | Historic SQL service-account regex; repeatable | — | -| `--historic-sql-redaction-pattern ` | Historic SQL SQL-literal redaction regex; repeatable | — | - -### Context Source Configuration - -| Flag | Description | Default | -|------|-------------|---------| -| `--source ` | Source connector type (`dbt`, `metricflow`, `metabase`, `looker`, `lookml`, `notion`) | — | -| `--source-connection-id ` | Connection id for source setup | — | -| `--source-path ` | Local source path for dbt, MetricFlow, or LookML | — | -| `--source-git-url ` | Git URL for dbt, MetricFlow, or LookML | — | -| `--source-branch ` | Git branch for source setup | — | -| `--source-subpath ` | Repo subpath for source setup | — | -| `--source-auth-token-ref ` | `env:` or `file:` credential ref for source repo auth | — | -| `--source-url ` | Source service URL for Metabase or Looker | — | -| `--source-api-key-ref ` | `env:` or `file:` API key ref for Metabase or Notion | — | -| `--source-client-id ` | Looker client id | — | -| `--source-client-secret-ref ` | `env:` or `file:` Looker client secret ref | — | -| `--source-warehouse-connection-id ` | Mapped warehouse connection id | — | -| `--source-project-name ` | dbt project name override | — | -| `--source-profiles-path ` | dbt profiles path | — | -| `--source-target ` | dbt target or source-specific mapping target | — | -| `--metabase-database-id ` | Metabase database id to map | — | -| `--notion-crawl-mode ` | Notion crawl mode (`all_accessible` or `selected_roots`) | — | -| `--notion-root-page-id ` | Notion root page id; repeatable | — | -| `--skip-initial-source-ingest` | Validate source setup without building source context during setup | `false` | -| `--skip-sources` | Mark optional source setup complete with no sources | `false` | +The setup wizard is the public configuration interface. It prompts for LLM +credentials, embeddings, database connections, context sources, Historic SQL, +and agent integration when those values are needed. ## Examples @@ -106,17 +39,8 @@ ktx setup [options] # Run the interactive setup wizard ktx setup -# Create a new project and run setup -ktx setup --new - -# Resume setup in an existing project -ktx setup --existing - -# Non-interactive setup with Anthropic key from environment -ktx setup --yes --anthropic-api-key-env ANTHROPIC_API_KEY - -# Set up a Postgres connection -ktx setup --database postgres --database-url "env:DATABASE_URL" +# Run setup for a specific project directory +ktx setup --project-dir ./analytics # Install agent integration for Claude Code only ktx setup --agents --target claude-code @@ -124,12 +48,6 @@ ktx setup --agents --target claude-code # Install agent integration globally for Codex ktx setup --agents --target codex --global -# Add a dbt source from a local path -ktx setup --source dbt --source-path ./my-dbt-project - -# Skip optional steps for a minimal setup -ktx setup --skip-sources --skip-agents - # Check setup readiness ktx status ``` @@ -156,5 +74,5 @@ Agent integration ready: yes (codex:project) |-------|-------|----------| | Setup resumes an unexpected project | `KTX_PROJECT_DIR` or nearest `ktx.yaml` points to another directory | Pass `--project-dir ` explicitly | | Health check for model fails | Provider key or model id is invalid | Set the correct environment variable or secret file and rerun setup | -| Setup cannot run in CI | Interactive prompts need a TTY | Use `--yes --no-input` with explicit flags for required values | +| Setup cannot run in CI | Interactive prompts need a TTY | Run setup interactively before CI, or provide a fixture `ktx.yaml` for automated tests | | Agent integration missing | Setup skipped the agents step | Run `ktx setup --agents --target ` | diff --git a/docs-site/content/docs/getting-started/quickstart.mdx b/docs-site/content/docs/getting-started/quickstart.mdx index 7aba00fd..635c666b 100644 --- a/docs-site/content/docs/getting-started/quickstart.mdx +++ b/docs-site/content/docs/getting-started/quickstart.mdx @@ -242,7 +242,7 @@ Agent integration ready: yes (claude-code:project) | Local embeddings hang or fail | The managed Python runtime cannot start or the local model runtime is unavailable | Install `uv`, run `ktx dev runtime status`, then run `ktx dev runtime install --feature local-embeddings --yes` and rerun setup | | Database connection test fails | Credentials, network access, warehouse, database, or schema value is wrong | Test the same URL with the database's native client, then rerun `ktx setup` and reconfigure the connection | | `KTX context built: no` in `ktx status` | Setup saved configuration but did not build context | Run `ktx setup` and choose to build context now | -| Agent integration is incomplete | Setup skipped the agents step or the target was not installed | Run `ktx setup --agents --target codex --project` using the target you need | +| Agent integration is incomplete | Setup skipped the agents step or the target was not installed | Run `ktx setup --agents --target codex` using the target you need | ## Next steps diff --git a/docs-site/content/docs/integrations/primary-sources.mdx b/docs-site/content/docs/integrations/primary-sources.mdx index 716f9f8d..46fa3716 100644 --- a/docs-site/content/docs/integrations/primary-sources.mdx +++ b/docs-site/content/docs/integrations/primary-sources.mdx @@ -37,7 +37,7 @@ The most full-featured connector. Supports schema introspection, foreign key det connections: my-postgres: driver: postgres - url: postgresql://user:password@host:5432/database + url: env:DATABASE_URL schema: public ``` @@ -326,7 +326,7 @@ Standard MySQL/MariaDB connector with full foreign key support and schema intros connections: my-mysql: driver: mysql - url: mysql://user:password@host:3306/database + url: env:MYSQL_DATABASE_URL ``` Or with individual fields: @@ -383,7 +383,7 @@ Connects to Microsoft SQL Server and Azure SQL. Supports multi-schema scanning w connections: my-sqlserver: driver: sqlserver - url: mssql://user:password@host:1433/database?trustServerCertificate=true + url: env:SQLSERVER_DATABASE_URL ``` Or with individual fields: diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-cross-dialect-readiness.md b/docs/superpowers/plans/2026-05-11-historic-sql-cross-dialect-readiness.md index a7a5cc6c..3fc3e496 100644 --- a/docs/superpowers/plans/2026-05-11-historic-sql-cross-dialect-readiness.md +++ b/docs/superpowers/plans/2026-05-11-historic-sql-cross-dialect-readiness.md @@ -40,37 +40,37 @@ This plan does not update `examples/postgres-historic/README.md` or `examples/po Modify: -- `packages/context/src/ingest/adapters/historic-sql/types.ts` +- `packages/context/src/ingest/adapters/historic-sql/types.ts` Adds optional probe `info` notes and lets injected historic-SQL dependencies use any reader/query client pair while preserving the existing Postgres-specific option. -- `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.ts` +- `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.ts` Moves low `pg_stat_statements.max` from `warnings` to `info`. -- `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.test.ts` Locks `track = none` as warning and low `max` as info. -- `packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.ts` +- `packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.ts` Returns `{ warnings: [], info: [] }` from `probe()`. -- `packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.test.ts` Locks the BigQuery probe return object. -- `packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.ts` +- `packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.ts` Returns `{ warnings: [], info: [] }` from `probe()`. -- `packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.test.ts` Locks the Snowflake probe return object. -- `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts` Updates test readers to return the normalized probe shape. -- `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts` Updates test readers to return the normalized probe shape. -- `packages/context/src/ingest/local-adapters.ts` +- `packages/context/src/ingest/local-adapters.ts` Accepts generic historic-SQL reader/query-client dependencies while keeping `postgresQueryClient` as the compatibility input used by current callers. -- `packages/context/src/ingest/local-adapters.test.ts` +- `packages/context/src/ingest/local-adapters.test.ts` Verifies generic reader/query-client injection and the existing Postgres compatibility path. -- `packages/cli/src/local-adapters.ts` +- `packages/cli/src/local-adapters.ts` Chooses Postgres, BigQuery, or Snowflake historic-SQL readers/query clients from the configured connection. -- `packages/cli/src/local-adapters.test.ts` +- `packages/cli/src/local-adapters.test.ts` Adds direct tests for CLI local adapter registration for Postgres, BigQuery, and Snowflake. -- `packages/cli/src/historic-sql-doctor.ts` +- `packages/cli/src/historic-sql-doctor.ts` Treats info-only Postgres probe notes as a passing doctor check, and warnings as warnings. -- `packages/cli/src/historic-sql-doctor.test.ts` +- `packages/cli/src/historic-sql-doctor.test.ts` Verifies low `pg_stat_statements.max` is pass/detail, while `track = none` remains warn. -- `packages/cli/src/doctor.test.ts` +- `packages/cli/src/doctor.test.ts` Updates the project doctor integration expectation for the new info-only behavior. ## Task 1: Normalize Historic-SQL Probe Results diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-end-to-end-retrieval-acceptance.md b/docs/superpowers/plans/2026-05-11-historic-sql-end-to-end-retrieval-acceptance.md index c9c40fd9..106131ed 100644 --- a/docs/superpowers/plans/2026-05-11-historic-sql-end-to-end-retrieval-acceptance.md +++ b/docs/superpowers/plans/2026-05-11-historic-sql-end-to-end-retrieval-acceptance.md @@ -44,7 +44,7 @@ Remaining acceptance gap this plan covers: Create: -- `packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts` Owns the end-to-end local regression for the redesigned historic-SQL pipeline. It uses the real adapter and local ingest runner, with fake deterministic reader/analysis/agent components so the test does not need a live database or LLM provider. ## Task 1: Add Real-Adapter Local Ingest Acceptance Coverage diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-foundations.md b/docs/superpowers/plans/2026-05-11-historic-sql-foundations.md index fdd97d3f..6705d56f 100644 --- a/docs/superpowers/plans/2026-05-11-historic-sql-foundations.md +++ b/docs/superpowers/plans/2026-05-11-historic-sql-foundations.md @@ -41,50 +41,50 @@ The next plan after this one should cover search enrichment from spec §6.2.3-§ Create: -- `packages/context/src/ingest/adapters/historic-sql/skill-schemas.ts` +- `packages/context/src/ingest/adapters/historic-sql/skill-schemas.ts` Owns the shared zod schemas for historic-SQL LLM outputs. -- `packages/context/src/ingest/adapters/historic-sql/skill-schemas.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/skill-schemas.test.ts` Locks schema acceptance, JSON schema generation, and future-key tolerance. -- `python/ktx-daemon/src/ktx_daemon/sql_analysis.py` +- `python/ktx-daemon/src/ktx_daemon/sql_analysis.py` Implements batch sqlglot parsing for table and clause-level column extraction. -- `python/ktx-daemon/tests/test_sql_analysis.py` +- `python/ktx-daemon/tests/test_sql_analysis.py` Tests batch parser behavior without FastAPI. Modify: -- `packages/context/src/ingest/index.ts` +- `packages/context/src/ingest/index.ts` Exports the new historic-SQL skill schemas. -- `packages/context/src/sl/types.ts` +- `packages/context/src/sl/types.ts` Adds `usage?: TableUsageOutput` to `SemanticLayerSource`. -- `packages/context/src/sl/schemas.ts` +- `packages/context/src/sl/schemas.ts` Accepts `usage` in standalone and overlay semantic-layer source validation. -- `packages/context/src/sl/semantic-layer.service.ts` +- `packages/context/src/sl/semantic-layer.service.ts` Projects manifest `usage` onto `SemanticLayerSource` and composes overlay usage intentionally. -- `packages/context/src/sl/semantic-layer.service.test.ts` +- `packages/context/src/sl/semantic-layer.service.test.ts` Tests source schema acceptance, manifest projection, and overlay composition. -- `packages/context/src/ingest/adapters/live-database/manifest.ts` +- `packages/context/src/ingest/adapters/live-database/manifest.ts` Adds `LiveDatabaseManifestTableEntry.usage`, existing-usage inputs, and `mergeUsagePreservingExternal()`. -- `packages/context/src/ingest/adapters/live-database/manifest.test.ts` +- `packages/context/src/ingest/adapters/live-database/manifest.test.ts` Tests scan-managed usage replacement while preserving external keys. -- `packages/context/src/scan/local-enrichment-artifacts.ts` +- `packages/context/src/scan/local-enrichment-artifacts.ts` Loads existing manifest usage and passes it through scan manifest rebuilds. -- `packages/context/src/scan/local-enrichment-artifacts.test.ts` +- `packages/context/src/scan/local-enrichment-artifacts.test.ts` Tests that structural scan rewrites preserve existing usage. -- `python/ktx-daemon/src/ktx_daemon/app.py` +- `python/ktx-daemon/src/ktx_daemon/app.py` Registers `/sql/analyze-batch`. -- `python/ktx-daemon/tests/test_app.py` +- `python/ktx-daemon/tests/test_app.py` Tests the FastAPI endpoint. -- `packages/context/src/sql-analysis/ports.ts` +- `packages/context/src/sql-analysis/ports.ts` Adds batch analysis types and `SqlAnalysisPort.analyzeBatch()`. -- `packages/context/src/sql-analysis/index.ts` +- `packages/context/src/sql-analysis/index.ts` Exports the new batch analysis types. -- `packages/context/src/sql-analysis/http-sql-analysis-port.ts` +- `packages/context/src/sql-analysis/http-sql-analysis-port.ts` Maps `/sql/analyze-batch` request and response payloads. -- `packages/context/src/sql-analysis/http-sql-analysis-port.test.ts` +- `packages/context/src/sql-analysis/http-sql-analysis-port.test.ts` Tests HTTP mapping and malformed response rejection. -- `packages/cli/src/managed-python-http.test.ts` +- `packages/cli/src/managed-python-http.test.ts` Verifies the managed daemon wrapper routes `analyzeBatch()`. -- Existing test files with `SqlAnalysisPort` object literals +- Existing test files with `SqlAnalysisPort` object literals Add a no-op `analyzeBatch: async () => new Map()` while legacy paths still use `analyzeForFingerprint()`. ## Task 1: Add Historic SQL Skill Schemas diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-pattern-shard-smoke-docs.md b/docs/superpowers/plans/2026-05-11-historic-sql-pattern-shard-smoke-docs.md index 9e386a16..b5382ff4 100644 --- a/docs/superpowers/plans/2026-05-11-historic-sql-pattern-shard-smoke-docs.md +++ b/docs/superpowers/plans/2026-05-11-historic-sql-pattern-shard-smoke-docs.md @@ -39,13 +39,13 @@ Remaining gap this plan fixes: ## File Structure -- Modify `scripts/examples-docs.test.mjs` +- Modify `scripts/examples-docs.test.mjs` Pins docs and smoke script to the sharded pattern WorkUnit contract. -- Modify `examples/postgres-historic/scripts/smoke.sh` +- Modify `examples/postgres-historic/scripts/smoke.sh` Validates `patterns-input/part-*.json` shard files and `historic-sql-patterns-part-*` stage-only WorkUnits. -- Modify `examples/postgres-historic/README.md` +- Modify `examples/postgres-historic/README.md` Documents `patterns-input.json` as the full audit artifact and `patterns-input/part-*.json` as bounded pattern WorkUnit input. -- Modify `examples/README.md` +- Modify `examples/README.md` Updates the short example catalog entry with the same audit-vs-shard wording. ### Task 1: Pin Example Tests To Pattern Shards diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-pattern-workunit-sharding.md b/docs/superpowers/plans/2026-05-11-historic-sql-pattern-workunit-sharding.md index ee7604a7..c67f6d78 100644 --- a/docs/superpowers/plans/2026-05-11-historic-sql-pattern-workunit-sharding.md +++ b/docs/superpowers/plans/2026-05-11-historic-sql-pattern-workunit-sharding.md @@ -30,23 +30,23 @@ No existing spec-derived plan is currently unimplemented in this worktree. This ## File Structure -- Create `packages/context/src/ingest/adapters/historic-sql/pattern-inputs.ts` +- Create `packages/context/src/ingest/adapters/historic-sql/pattern-inputs.ts` Owns deterministic pattern audit ordering, cross-table candidate filtering, byte-bounded shard creation, shard path constants, and shard path detection. -- Create `packages/context/src/ingest/adapters/historic-sql/pattern-inputs.test.ts` +- Create `packages/context/src/ingest/adapters/historic-sql/pattern-inputs.test.ts` Covers deterministic shard ordering, single-table exclusion from WorkUnit shards, byte limits, and oversize-template manifest warnings. -- Modify `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts` +- Modify `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts` Writes full `patterns-input.json` plus bounded `patterns-input/part-0001.json` shard files, and appends shard warnings to `manifest.json`. -- Modify `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts` +- Modify `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts` Adds a regression for audit file preservation and sharded WorkUnit input creation. -- Modify `packages/context/src/ingest/adapters/historic-sql/chunk-unified.ts` +- Modify `packages/context/src/ingest/adapters/historic-sql/chunk-unified.ts` Emits one patterns WorkUnit per changed shard path, treats root `patterns-input.json` as audit-only, and includes shard paths in the scope descriptor and eviction calculation. -- Modify `packages/context/src/ingest/adapters/historic-sql/chunk-unified.test.ts` +- Modify `packages/context/src/ingest/adapters/historic-sql/chunk-unified.test.ts` Updates root-file expectations and adds multi-shard diff behavior. -- Modify `packages/context/skills/historic_sql_patterns/SKILL.md` +- Modify `packages/context/skills/historic_sql_patterns/SKILL.md` Tells the skill to read the exact pattern shard in `rawFiles` and emit evidence with that shard as `rawPath`. -- Modify `packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts` +- Modify `packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts` Updates the fake agent to emit pattern evidence for `historic-sql-patterns-part-0001`. -- Modify `packages/context/src/ingest/ingest-runtime-assets.test.ts` +- Modify `packages/context/src/ingest/ingest-runtime-assets.test.ts` Keeps packaged skill assertions aligned with sharded pattern file guidance. ## Task 1: Add Pattern Input Sharding Helper diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-redaction-hardening.md b/docs/superpowers/plans/2026-05-11-historic-sql-redaction-hardening.md index 1adcdfd3..e59e164b 100644 --- a/docs/superpowers/plans/2026-05-11-historic-sql-redaction-hardening.md +++ b/docs/superpowers/plans/2026-05-11-historic-sql-redaction-hardening.md @@ -55,16 +55,16 @@ Remaining spec gap this plan covers: Create: -- `packages/context/src/ingest/adapters/historic-sql/redaction.ts` +- `packages/context/src/ingest/adapters/historic-sql/redaction.ts` Owns compilation and application of historic-SQL SQL-text redaction patterns. Supports JavaScript regex strings and the documented `(?i)` case-insensitive prefix used by setup tests/docs. -- `packages/context/src/ingest/adapters/historic-sql/redaction.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/redaction.test.ts` Tests raw regex replacement, `(?i)` compatibility, empty config behavior, and invalid-pattern diagnostics. Modify: -- `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts` +- `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts` Compiles `config.redactionPatterns` once per fetch. Keeps original SQL for filtering and `SqlAnalysisPort.analyzeBatch()`, then stores redacted SQL in `ParsedTemplate.template.canonicalSql` before `toStagedTable()` and `toPatternsInput()` serialize files. -- `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts` Adds a regression proving raw secrets are absent from staged artifacts while `analyzeBatch()` still receives the original SQL. ## Task 1: Add Historic SQL Redaction Helper @@ -89,7 +89,7 @@ describe('historic-SQL redaction', () => { ]); const sql = - "select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; + "select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; // pragma: allowlist secret expect(redactHistoricSqlText(sql, redactors)).toBe( "select * from public.api_events where api_key = '[REDACTED]' and note = '[REDACTED]'", @@ -202,7 +202,7 @@ Append this test inside the existing `describe('stageHistoricSqlAggregatedSnapsh it('redacts configured SQL substrings in staged artifacts while analyzing original SQL', async () => { const stagedDir = await tempDir(); const originalSql = - "select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; + "select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; // pragma: allowlist secret const reader: HistoricSqlReader = { async probe() { return { warnings: [], info: [] }; diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-search-enrichment.md b/docs/superpowers/plans/2026-05-11-historic-sql-search-enrichment.md index ee960bb8..cafc234b 100644 --- a/docs/superpowers/plans/2026-05-11-historic-sql-search-enrichment.md +++ b/docs/superpowers/plans/2026-05-11-historic-sql-search-enrichment.md @@ -37,27 +37,27 @@ This plan does not rewrite the historic-SQL adapter, readers, skills, projection Modify: -- `packages/context/src/sl/sl-search.service.ts` +- `packages/context/src/sl/sl-search.service.ts` Adds usage narrative, frequency, filters, group-bys, joins, and stale marker to the canonical SL search text. Preserves snippets returned by repository search for direct `SlSearchService.search()` callers. -- `packages/context/src/sl/sl-search.service.test.ts` +- `packages/context/src/sl/sl-search.service.test.ts` Tests usage search-text content and direct service snippet pass-through. -- `packages/context/src/sl/ports.ts` +- `packages/context/src/sl/ports.ts` Extends `SlSourcesIndexPort.search()` rows with optional `snippet`. -- `packages/context/src/sl/sqlite-sl-sources-index.ts` +- `packages/context/src/sl/sqlite-sl-sources-index.ts` Adds FTS5 `snippet()` selection to lexical candidate search and direct index search. -- `packages/context/src/sl/sqlite-sl-sources-index.test.ts` +- `packages/context/src/sl/sqlite-sl-sources-index.test.ts` Locks snippet behavior for both direct search and lexical lane candidates. -- `packages/context/src/sl/local-sl.ts` +- `packages/context/src/sl/local-sl.ts` Adds `frequencyTier` and `snippet` to query-mode `LocalSlSourceSearchResult`; collects snippets from the lexical lane and hydrates frequency from `SemanticLayerSource.usage`. -- `packages/context/src/sl/local-sl.test.ts` +- `packages/context/src/sl/local-sl.test.ts` Tests that usage-only terms can find a source and that results include `frequencyTier` and FTS snippet. -- `packages/context/src/sl/pglite-sl-search-prototype.ts` +- `packages/context/src/sl/pglite-sl-search-prototype.ts` Propagates `frequencyTier` for the prototype backend so the shared result type stays truthful. -- `packages/context/src/mcp/types.ts` +- `packages/context/src/mcp/types.ts` Adds `frequencyTier` and `snippet` to `KtxSemanticLayerSourceSummary`. -- `packages/context/src/mcp/local-project-ports.ts` +- `packages/context/src/mcp/local-project-ports.ts` Includes `frequencyTier` and `snippet` in `semanticLayer.listSources()` output. -- `packages/context/src/mcp/local-project-ports.test.ts` +- `packages/context/src/mcp/local-project-ports.test.ts` Tests the agent/MCP-facing list response. ## Task 1: Index Historic SQL Usage In SL Search Text diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-skills-projection-cutover.md b/docs/superpowers/plans/2026-05-11-historic-sql-skills-projection-cutover.md index a892542e..a7494e2d 100644 --- a/docs/superpowers/plans/2026-05-11-historic-sql-skills-projection-cutover.md +++ b/docs/superpowers/plans/2026-05-11-historic-sql-skills-projection-cutover.md @@ -52,58 +52,58 @@ Still not implemented: Create: -- `packages/context/src/ingest/adapters/historic-sql/evidence.ts` +- `packages/context/src/ingest/adapters/historic-sql/evidence.ts` Owns typed evidence envelopes, ignored evidence path helpers, and load/write helpers for table usage and pattern evidence. -- `packages/context/src/ingest/adapters/historic-sql/evidence.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/evidence.test.ts` Tests evidence schema validation, path normalization, and loader rejection of malformed evidence. -- `packages/context/src/ingest/adapters/historic-sql/evidence-tool.ts` +- `packages/context/src/ingest/adapters/historic-sql/evidence-tool.ts` Adds `emit_historic_sql_evidence`, the only write tool the two new historic-SQL skills use. -- `packages/context/src/ingest/adapters/historic-sql/evidence-tool.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/evidence-tool.test.ts` Tests the tool writes ignored run-local JSON with `skipLock: true` and rejects non-historic ingest sessions. -- `packages/context/src/ingest/adapters/historic-sql/projection.ts` +- `packages/context/src/ingest/adapters/historic-sql/projection.ts` Projects table usage evidence into manifest shards, writes pattern wiki pages, marks stale usage/pages, and deletes legacy query pages. -- `packages/context/src/ingest/adapters/historic-sql/projection.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/projection.test.ts` Tests `_schema` merge, stale usage, pattern slug reuse, stale page tagging, archive movement, and legacy page cleanup. -- `packages/context/src/ingest/adapters/historic-sql/post-processor.ts` +- `packages/context/src/ingest/adapters/historic-sql/post-processor.ts` Implements `IngestBundlePostProcessorPort` for the deterministic projection phase. -- `packages/context/src/ingest/adapters/historic-sql/post-processor.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/post-processor.test.ts` Tests post-processor path resolution from `workdir`, `connectionId`, `sourceKey`, and `syncId`. -- `packages/context/skills/historic_sql_table_digest/SKILL.md` +- `packages/context/skills/historic_sql_table_digest/SKILL.md` Skill for one changed `tables/*.json` WorkUnit; emits one table usage evidence object. -- `packages/context/skills/historic_sql_patterns/SKILL.md` +- `packages/context/skills/historic_sql_patterns/SKILL.md` Skill for `patterns-input.json`; emits one pattern evidence object per recurring cross-table intent. Modify: -- `packages/context/src/ingest/adapters/historic-sql/types.ts` +- `packages/context/src/ingest/adapters/historic-sql/types.ts` Keep only unified config/staged schemas and reader contracts; extend config preprocessing for existing `serviceAccountUserPatterns` and `minCalls` aliases. -- `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts` +- `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts` Add `staleArchiveAfterDays` to `manifest.json` so projection can archive stale pattern pages deterministically. -- `packages/context/src/ingest/adapters/historic-sql/chunk-unified.ts` +- `packages/context/src/ingest/adapters/historic-sql/chunk-unified.ts` Keep the same WorkUnits, but mention `emit_historic_sql_evidence` in `notes`. -- `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.ts` +- `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.ts` Switch production fetch/chunk/scope to the unified hot path, replace skills, remove legacy triage support, and run legacy PGSS baseline cache cleanup. -- `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts` Rewrite around unified staging and new skills. -- `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.ts` +- `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.ts` Inline the PGSS probe logic so `postgres-pgss-query-history-reader.ts` can be deleted. -- `packages/context/src/ingest/local-adapters.ts` +- `packages/context/src/ingest/local-adapters.ts` Use `PostgresPgssReader` for local Postgres historic SQL and return unified pull config. -- `packages/context/src/ingest/local-bundle-runtime.ts` +- `packages/context/src/ingest/local-bundle-runtime.ts` Add the source-specific evidence tool to historic-SQL WorkUnits and register the historic-SQL post-processor. -- `packages/context/src/ingest/ingest-runtime-assets.test.ts` +- `packages/context/src/ingest/ingest-runtime-assets.test.ts` Replace old skill asset assertions with the two new skills. -- `packages/context/src/memory/memory-runtime-assets.test.ts` +- `packages/context/src/memory/memory-runtime-assets.test.ts` Replace old historic-SQL skill heading with the two new skill headings. -- `packages/context/src/package-exports.test.ts` +- `packages/context/src/package-exports.test.ts` Remove legacy export assertions and add evidence/projection export assertions. -- `packages/context/src/ingest/index.ts` +- `packages/context/src/ingest/index.ts` Export new evidence/projection/post-processor helpers and remove legacy historic-SQL exports. -- `packages/cli/src/setup-databases.ts` and `packages/cli/src/historic-sql-doctor.ts` +- `packages/cli/src/setup-databases.ts` and `packages/cli/src/historic-sql-doctor.ts` Import `PostgresPgssReader` instead of `PostgresPgssQueryHistoryReader`. -- `packages/cli/src/commands/setup-commands.ts`, `packages/cli/src/index.test.ts`, `packages/cli/src/setup-databases.test.ts` +- `packages/cli/src/commands/setup-commands.ts`, `packages/cli/src/index.test.ts`, `packages/cli/src/setup-databases.test.ts` Rename generated config to `minExecutions` while accepting the old `--historic-sql-min-calls` flag for one release. -- `packages/context/prompts/skills/page_triage_classifier.md`, `packages/context/src/ingest/page-triage/page-triage.service.test.ts`, `packages/context/src/ingest/ingest-prompts.test.ts` +- `packages/context/prompts/skills/page_triage_classifier.md`, `packages/context/src/ingest/page-triage/page-triage.service.test.ts`, `packages/context/src/ingest/ingest-prompts.test.ts` Remove historic-SQL template triage examples because the new adapter no longer uses page triage. Delete: diff --git a/packages/cli/src/commands/setup-commands.ts b/packages/cli/src/commands/setup-commands.ts index 1688724d..3da8d094 100644 --- a/packages/cli/src/commands/setup-commands.ts +++ b/packages/cli/src/commands/setup-commands.ts @@ -64,13 +64,6 @@ function sourceType(value: string): KtxSetupSourceType { throw new InvalidArgumentError(`invalid choice '${value}'`); } -function agentScope(value: string): 'project' | 'global' { - if (value === 'project' || value === 'global') { - return value; - } - throw new InvalidArgumentError(`invalid choice '${value}'`); -} - function positiveNumber(value: string): number { const parsed = Number.parseInt(value, 10); if (!Number.isInteger(parsed) || parsed <= 0) { @@ -97,7 +90,6 @@ function shouldShowSetupEntryMenu( agents?: boolean; target?: string; global?: boolean; - project?: boolean; skipAgents?: boolean; yes?: boolean; input?: boolean; @@ -142,7 +134,6 @@ function shouldShowSetupEntryMenu( metabaseDatabaseId?: number; notionCrawlMode?: string; notionRootPageId?: string[]; - skipInitialSourceIngest?: boolean; skipSources?: boolean; }, command: Command, @@ -172,7 +163,6 @@ function shouldShowSetupEntryMenu( 'agents', 'target', 'global', - 'project', 'skipAgents', 'yes', 'input', @@ -211,7 +201,6 @@ function shouldShowSetupEntryMenu( 'sourceTarget', 'metabaseDatabaseId', 'notionCrawlMode', - 'skipInitialSourceIngest', 'skipSources', ].some((optionName) => optionWasSpecified(command, optionName)); } @@ -220,9 +209,9 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo const setup = program .command('setup') .description('Set up or resume a local KTX project') - .option('--project-dir ', 'KTX project directory') - .option('--new', 'Create a new KTX project before setup', false) - .option('--existing', 'Use an existing KTX project', false) + .addOption(new Option('--project-dir ', 'KTX project directory').hideHelp()) + .addOption(new Option('--new', 'Create a new KTX project before setup').hideHelp().default(false)) + .addOption(new Option('--existing', 'Use an existing KTX project').hideHelp().default(false)) .option('--agents', 'Install agent integration only', false) .addOption( new Option('--target ', 'Agent target').choices([ @@ -233,94 +222,124 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo 'universal', ]), ) - .addOption(new Option('--agent-scope ', 'Agent install scope').argParser(agentScope).default('project')) - .option('--project', 'Install agent integration into the project scope', false) .option('--global', 'Install agent integration into the global target scope', false) - .option('--skip-agents', 'Leave agent integration incomplete for now', false) + .addOption(new Option('--skip-agents', 'Leave agent integration incomplete for now').hideHelp().default(false)) .option('--yes', 'Accept safe defaults in non-interactive setup', false) .option('--no-input', 'Disable interactive terminal input') - .addOption(new Option('--llm-backend ', 'LLM backend').argParser(llmBackend)) - .option('--anthropic-api-key-env ', 'Environment variable containing the Anthropic API key') - .option('--anthropic-api-key-file ', 'File containing the Anthropic API key') - .option('--anthropic-model ', 'Anthropic model ID to validate and save') - .option('--vertex-project ', 'Google Vertex AI project ID, env:NAME, or file:/path') - .option('--vertex-location ', 'Google Vertex AI location, env:NAME, or file:/path') - .addOption(new Option('--skip-llm', 'Leave LLM setup incomplete for now').hideHelp().default(false)) - .addOption(new Option('--embedding-backend ', 'Embedding backend').argParser(embeddingBackend)) - .option('--embedding-api-key-env ', 'Environment variable containing the embedding provider API key') - .option('--embedding-api-key-file ', 'File containing the embedding provider API key') - .addOption(new Option('--skip-embeddings', 'Leave embedding setup incomplete for now').hideHelp().default(false)) - .option( - '--database ', - 'Database driver to configure; repeatable', - (value, previous: KtxSetupDatabaseDriver[]) => { - return [...previous, databaseDriver(value)]; - }, - [] as KtxSetupDatabaseDriver[], - ) - .option( - '--database-connection-id ', - 'Existing selected connection id or new connection id', - (value, previous: string[]) => [...previous, value], - [], - ) - .option('--new-database-connection-id ', 'Connection id for one new database connection', (value) => { - if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(value)) { - throw new InvalidArgumentError(`Unsafe connection id: ${value}`); - } - return value; - }) - .option('--database-url ', 'URL, env:NAME, or file:/path for one new URL-style database connection') - .option( - '--database-schema ', - 'Database schema to include; repeatable', - (value, previous: string[]) => [...previous, value], - [], - ) - .option('--enable-historic-sql', 'Enable Historic SQL when the selected database supports it', false) - .option('--disable-historic-sql', 'Disable Historic SQL for the selected database', false) - .option('--historic-sql-window-days ', 'Historic SQL query-history window', positiveInteger) - .option('--historic-sql-min-executions ', 'Minimum Historic SQL executions for a template', positiveInteger) - .option( - '--historic-sql-service-account-pattern ', - 'Historic SQL service-account regex; repeatable', - (value, previous: string[]) => [...previous, value], - [], - ) - .option( - '--historic-sql-redaction-pattern ', - 'Historic SQL SQL-literal redaction regex; repeatable', - (value, previous: string[]) => [...previous, value], - [], - ) - .option('--skip-databases', 'Leave database setup incomplete; KTX cannot work until a primary source is added', false) - .addOption(new Option('--source ', 'Source connector type').argParser(sourceType)) - .option('--source-connection-id ', 'Connection id for source setup') - .option('--source-path ', 'Local source path for dbt, MetricFlow, or LookML') - .option('--source-git-url ', 'Git URL for dbt, MetricFlow, or LookML') - .option('--source-branch ', 'Git branch for source setup') - .option('--source-subpath ', 'Repo subpath for source setup') - .option('--source-auth-token-ref ', 'env: or file: credential ref for source repo auth') - .option('--source-url ', 'Source service URL for Metabase or Looker') - .option('--source-api-key-ref ', 'env: or file: API key ref for Metabase or Notion') - .option('--source-client-id ', 'Looker client id') - .option('--source-client-secret-ref ', 'env: or file: Looker client secret ref') - .option('--source-warehouse-connection-id ', 'Mapped warehouse connection id') - .option('--source-project-name ', 'dbt project name override') - .option('--source-profiles-path ', 'dbt profiles path') - .option('--source-target ', 'dbt target or source-specific mapping target') - .option('--metabase-database-id ', 'Metabase database id to map', positiveNumber) + .addOption(new Option('--llm-backend ', 'LLM backend').argParser(llmBackend).hideHelp()) .addOption( - new Option('--notion-crawl-mode ', 'Notion crawl mode').choices(['all_accessible', 'selected_roots']), + new Option('--anthropic-api-key-env ', 'Environment variable containing the Anthropic API key').hideHelp(), ) - .option( - '--notion-root-page-id ', - 'Notion root page id; repeatable', - (value, previous: string[]) => [...previous, value], - [], + .addOption( + new Option('--anthropic-api-key-file ', 'File containing the Anthropic API key').hideHelp(), ) - .option('--skip-initial-source-ingest', 'Validate source setup without building source context during setup', false) - .option('--skip-sources', 'Mark optional source setup complete with no sources', false) + .addOption(new Option('--anthropic-model ', 'Anthropic model ID to validate and save').hideHelp()) + .addOption(new Option('--vertex-project ', 'Google Vertex AI project ID, env:NAME, or file:/path').hideHelp()) + .addOption(new Option('--vertex-location ', 'Google Vertex AI location, env:NAME, or file:/path').hideHelp()) + .addOption(new Option('--skip-llm', 'Leave LLM setup incomplete for now').hideHelp().default(false)) + .addOption(new Option('--embedding-backend ', 'Embedding backend').argParser(embeddingBackend).hideHelp()) + .addOption( + new Option( + '--embedding-api-key-env ', + 'Environment variable containing the embedding provider API key', + ).hideHelp(), + ) + .addOption( + new Option('--embedding-api-key-file ', 'File containing the embedding provider API key').hideHelp(), + ) + .addOption(new Option('--skip-embeddings', 'Leave embedding setup incomplete for now').hideHelp().default(false)) + .addOption( + new Option('--database ', 'Database driver to configure; repeatable') + .argParser((value, previous: KtxSetupDatabaseDriver[]) => { + return [...previous, databaseDriver(value)]; + }) + .default([] as KtxSetupDatabaseDriver[]) + .hideHelp(), + ) + .addOption( + new Option('--database-connection-id ', 'Existing selected connection id or new connection id') + .argParser((value, previous: string[]) => [...previous, value]) + .default([] as string[]) + .hideHelp(), + ) + .addOption( + new Option('--new-database-connection-id ', 'Connection id for one new database connection') + .argParser((value) => { + if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(value)) { + throw new InvalidArgumentError(`Unsafe connection id: ${value}`); + } + return value; + }) + .hideHelp(), + ) + .addOption( + new Option('--database-url ', 'URL, env:NAME, or file:/path for one new URL-style database connection').hideHelp(), + ) + .addOption( + new Option('--database-schema ', 'Database schema to include; repeatable') + .argParser((value, previous: string[]) => [...previous, value]) + .default([] as string[]) + .hideHelp(), + ) + .addOption( + new Option('--enable-historic-sql', 'Enable Historic SQL when the selected database supports it') + .hideHelp() + .default(false), + ) + .addOption( + new Option('--disable-historic-sql', 'Disable Historic SQL for the selected database').hideHelp().default(false), + ) + .addOption(new Option('--historic-sql-window-days ', 'Historic SQL query-history window').argParser(positiveInteger).hideHelp()) + .addOption( + new Option('--historic-sql-min-executions ', 'Minimum Historic SQL executions for a template') + .argParser(positiveInteger) + .hideHelp(), + ) + .addOption( + new Option('--historic-sql-service-account-pattern ', 'Historic SQL service-account regex; repeatable') + .argParser((value, previous: string[]) => [...previous, value]) + .default([] as string[]) + .hideHelp(), + ) + .addOption( + new Option('--historic-sql-redaction-pattern ', 'Historic SQL SQL-literal redaction regex; repeatable') + .argParser((value, previous: string[]) => [...previous, value]) + .default([] as string[]) + .hideHelp(), + ) + .addOption( + new Option('--skip-databases', 'Leave database setup incomplete; KTX cannot work until a primary source is added') + .hideHelp() + .default(false), + ) + .addOption(new Option('--source ', 'Source connector type').argParser(sourceType).hideHelp()) + .addOption(new Option('--source-connection-id ', 'Connection id for source setup').hideHelp()) + .addOption(new Option('--source-path ', 'Local source path for dbt, MetricFlow, or LookML').hideHelp()) + .addOption(new Option('--source-git-url ', 'Git URL for dbt, MetricFlow, or LookML').hideHelp()) + .addOption(new Option('--source-branch ', 'Git branch for source setup').hideHelp()) + .addOption(new Option('--source-subpath ', 'Repo subpath for source setup').hideHelp()) + .addOption(new Option('--source-auth-token-ref ', 'env: or file: credential ref for source repo auth').hideHelp()) + .addOption(new Option('--source-url ', 'Source service URL for Metabase or Looker').hideHelp()) + .addOption(new Option('--source-api-key-ref ', 'env: or file: API key ref for Metabase or Notion').hideHelp()) + .addOption(new Option('--source-client-id ', 'Looker client id').hideHelp()) + .addOption(new Option('--source-client-secret-ref ', 'env: or file: Looker client secret ref').hideHelp()) + .addOption(new Option('--source-warehouse-connection-id ', 'Mapped warehouse connection id').hideHelp()) + .addOption(new Option('--source-project-name ', 'dbt project name override').hideHelp()) + .addOption(new Option('--source-profiles-path ', 'dbt profiles path').hideHelp()) + .addOption(new Option('--source-target ', 'dbt target or source-specific mapping target').hideHelp()) + .addOption(new Option('--metabase-database-id ', 'Metabase database id to map').argParser(positiveNumber).hideHelp()) + .addOption( + new Option('--notion-crawl-mode ', 'Notion crawl mode') + .choices(['all_accessible', 'selected_roots']) + .hideHelp(), + ) + .addOption( + new Option('--notion-root-page-id ', 'Notion root page id; repeatable') + .argParser((value, previous: string[]) => [...previous, value]) + .default([] as string[]) + .hideHelp(), + ) + .addOption(new Option('--skip-sources', 'Mark optional source setup complete with no sources').hideHelp().default(false)) .showHelpAfterError(); setup.hook('preAction', (_thisCommand, actionCommand) => { @@ -371,7 +390,7 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo } const mode = options.new ? 'new' : options.existing ? 'existing' : 'auto'; - const resolvedAgentScope = options.global ? 'global' : options.agentScope; + const resolvedAgentScope = options.global ? 'global' : 'project'; await runSetupArgs(context, { command: 'run', projectDir: resolveCommandProjectDir(command), diff --git a/packages/cli/src/commands/status-commands.ts b/packages/cli/src/commands/status-commands.ts index d834e15b..52032e59 100644 --- a/packages/cli/src/commands/status-commands.ts +++ b/packages/cli/src/commands/status-commands.ts @@ -16,8 +16,9 @@ export function registerStatusCommands(program: Command, context: KtxCliCommandC .command('status') .description('Check current KTX setup and project readiness') .option('--json', 'Print JSON output', false) + .option('-v, --verbose', 'Show every check, including passing ones', false) .option('--no-input', 'Disable interactive terminal input') - .action(async (options: { json?: boolean; input?: boolean }, command) => { + .action(async (options: { json?: boolean; verbose?: boolean; input?: boolean }, command) => { const runner = context.deps.doctor ?? (await import('../doctor.js')).runKtxDoctor; const explicitOrEnvProjectDir = resolveCommandProjectDirOverride(command); const nearestProjectDir = explicitOrEnvProjectDir ? undefined : findNearestKtxProjectDir(process.cwd()); @@ -27,6 +28,7 @@ export function registerStatusCommands(program: Command, context: KtxCliCommandC { command: 'setup', outputMode: outputMode(options), + verbose: options.verbose === true, ...inputMode(options), }, context.io, @@ -40,6 +42,7 @@ export function registerStatusCommands(program: Command, context: KtxCliCommandC command: 'project', projectDir: resolveCommandProjectDir(command), outputMode: outputMode(options), + verbose: options.verbose === true, ...inputMode(options), }, context.io, diff --git a/packages/cli/src/connection.test.ts b/packages/cli/src/connection.test.ts index 6eb3a08c..0d592b00 100644 --- a/packages/cli/src/connection.test.ts +++ b/packages/cli/src/connection.test.ts @@ -159,7 +159,7 @@ describe('runKtxConnection', () => { prod_metabase: { driver: 'metabase', api_url: 'http://metabase.example.test', - api_key: 'mb_test', + api_key: 'mb_test', // pragma: allowlist secret }, }); const testConnection = vi.fn(async () => ({ success: true as const })); diff --git a/packages/cli/src/database-tree-picker.test.ts b/packages/cli/src/database-tree-picker.test.ts new file mode 100644 index 00000000..5559ee42 --- /dev/null +++ b/packages/cli/src/database-tree-picker.test.ts @@ -0,0 +1,188 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + pickDatabaseScope, + type DatabaseTreePickerRenderer, + type PickDatabaseScopeArgs, +} from './database-tree-picker.js'; +import type { TreePickerChrome, TreePickerResult } from './tree-picker-tui.js'; +import type { PickerState } from './tree-picker-state.js'; + +function makeIo() { + let stdout = ''; + let stderr = ''; + return { + io: { + stdout: { isTTY: true, write: (chunk: string) => { stdout += chunk; } }, + stderr: { write: (chunk: string) => { stderr += chunk; } }, + }, + stdout: () => stdout, + stderr: () => stderr, + }; +} + +function captureRenderer(): { + renderer: DatabaseTreePickerRenderer; + capture: { chrome?: TreePickerChrome; state?: PickerState }; + setResult: (result: TreePickerResult) => void; +} { + const capture: { chrome?: TreePickerChrome; state?: PickerState } = {}; + let nextResult: TreePickerResult = { kind: 'quit' }; + const renderer: DatabaseTreePickerRenderer = vi.fn(async (chrome, state) => { + capture.chrome = chrome; + capture.state = state; + return nextResult; + }); + return { + renderer, + capture, + setResult: (result) => { + nextResult = result; + }, + }; +} + +const discovered = [ + { schema: 'analytics', name: 'customers', kind: 'table' as const }, + { schema: 'analytics', name: 'orders', kind: 'table' as const }, + { schema: 'public', name: 'events', kind: 'view' as const }, + { schema: 'public', name: 'sessions', kind: 'table' as const }, +]; + +function baseArgs(overrides: Partial = {}): PickDatabaseScopeArgs { + return { + connectionId: 'warehouse', + schemaNoun: 'schema', + schemaNounPlural: 'schemas', + discovered, + existing: { enabledTables: [] }, + defaultSchemas: ['analytics'], + supportsSchemaScope: true, + ...overrides, + }; +} + +describe('pickDatabaseScope', () => { + it('builds a 2-level tree (schemas as parents, tables as children) and uses save-empty action', async () => { + const { renderer, capture, setResult } = captureRenderer(); + setResult({ kind: 'quit' }); + + await pickDatabaseScope(baseArgs(), makeIo().io, renderer); + + expect(capture.state?.skipEmptyAction).toBe('save-empty'); + const schemaIds = capture.state?.tree.filter((n) => n.parentId === null).map((n) => n.id); + const tableIds = capture.state?.tree.filter((n) => n.parentId !== null).map((n) => n.id); + expect((schemaIds ?? []).sort()).toEqual(['analytics', 'public']); + expect((tableIds ?? []).sort()).toEqual([ + 'analytics.customers', + 'analytics.orders', + 'public.events', + 'public.sessions', + ]); + expect(capture.state?.byId.get('public.events')?.title).toBe('events (view)'); + }); + + it('pre-checks default schemas at the parent level when no existing selection', async () => { + const { renderer, capture, setResult } = captureRenderer(); + setResult({ kind: 'quit' }); + + await pickDatabaseScope(baseArgs({ defaultSchemas: ['analytics'] }), makeIo().io, renderer); + + expect([...(capture.state?.checked ?? [])]).toEqual(['analytics']); + }); + + it('collapses an existing full-schema selection back into the parent check', async () => { + const { renderer, capture, setResult } = captureRenderer(); + setResult({ kind: 'quit' }); + + await pickDatabaseScope( + baseArgs({ existing: { enabledTables: ['analytics.customers', 'analytics.orders'] } }), + makeIo().io, + renderer, + ); + + expect([...(capture.state?.checked ?? [])]).toEqual(['analytics']); + }); + + it('keeps a partial existing selection at the leaf level', async () => { + const { renderer, capture, setResult } = captureRenderer(); + setResult({ kind: 'quit' }); + + await pickDatabaseScope( + baseArgs({ existing: { enabledTables: ['analytics.customers'] } }), + makeIo().io, + renderer, + ); + + expect([...(capture.state?.checked ?? [])]).toEqual(['analytics.customers']); + }); + + it('expands a selected schema parent into all its tables and derives activeSchemas', async () => { + const { renderer, setResult } = captureRenderer(); + setResult({ kind: 'save', selectedIds: ['analytics'] }); + + const result = await pickDatabaseScope(baseArgs(), makeIo().io, renderer); + + expect(result).toEqual({ + kind: 'selected', + activeSchemas: ['analytics'], + enabledTables: ['analytics.customers', 'analytics.orders'], + }); + }); + + it('combines parent and individual leaf selections without duplicate tables', async () => { + const { renderer, setResult } = captureRenderer(); + setResult({ kind: 'save', selectedIds: ['analytics', 'public.events'] }); + + const result = await pickDatabaseScope(baseArgs(), makeIo().io, renderer); + + expect(result).toEqual({ + kind: 'selected', + activeSchemas: ['analytics', 'public'], + enabledTables: ['analytics.customers', 'analytics.orders', 'public.events'], + }); + }); + + it('treats empty save as enable-all', async () => { + const { renderer, setResult } = captureRenderer(); + setResult({ kind: 'save', selectedIds: [] }); + + const result = await pickDatabaseScope(baseArgs(), makeIo().io, renderer); + + expect(result).toEqual({ + kind: 'selected', + activeSchemas: ['analytics', 'public'], + enabledTables: [ + 'analytics.customers', + 'analytics.orders', + 'public.events', + 'public.sessions', + ], + }); + }); + + it('omits activeSchemas when the driver does not support a schema scope', async () => { + const { renderer, setResult } = captureRenderer(); + setResult({ kind: 'save', selectedIds: ['analytics'] }); + + const result = await pickDatabaseScope( + baseArgs({ supportsSchemaScope: false }), + makeIo().io, + renderer, + ); + + expect(result).toEqual({ + kind: 'selected', + activeSchemas: [], + enabledTables: ['analytics.customers', 'analytics.orders'], + }); + }); + + it('returns back when the picker quits', async () => { + const { renderer, setResult } = captureRenderer(); + setResult({ kind: 'quit' }); + + const result = await pickDatabaseScope(baseArgs(), makeIo().io, renderer); + + expect(result).toEqual({ kind: 'back' }); + }); +}); diff --git a/packages/cli/src/database-tree-picker.ts b/packages/cli/src/database-tree-picker.ts new file mode 100644 index 00000000..d494003d --- /dev/null +++ b/packages/cli/src/database-tree-picker.ts @@ -0,0 +1,210 @@ +import type { KtxTableListEntry } from '@ktx/context/scan'; +import type { KtxCliIo } from './cli-runtime.js'; +import { profileMark } from './startup-profile.js'; +import { + buildInitialState, + buildPickerTree, + type PickerState, + type TreePickerNode, + type TreePickerNodeInput, +} from './tree-picker-state.js'; +import { + renderTreePickerTui, + type TreePickerChrome, + type TreePickerResult, + type TreePickerTuiIo, +} from './tree-picker-tui.js'; + +profileMark('module:database-tree-picker'); + +const DATABASE_SCRIPTED_MODE_HINT = + 'Database picker requires a TTY. Use --no-input and the relevant flags for scripted mode.'; + +export type DatabaseTreePickerRenderer = ( + chrome: TreePickerChrome, + initialState: PickerState, + io: TreePickerTuiIo, +) => Promise; + +function defaultRenderer( + chrome: TreePickerChrome, + initialState: PickerState, + io: TreePickerTuiIo, +): Promise { + return renderTreePickerTui({ chrome, initialState }, io, { scriptedModeHint: DATABASE_SCRIPTED_MODE_HINT }); +} + +export type DatabaseScopePickResult = + | { kind: 'selected'; activeSchemas: string[]; enabledTables: string[] } + | { kind: 'back' }; + +export interface PickDatabaseScopeArgs { + connectionId: string; + schemaNoun: string; + schemaNounPlural: string; + discovered: readonly KtxTableListEntry[]; + existing: { enabledTables: readonly string[] }; + defaultSchemas: readonly string[]; + supportsSchemaScope: boolean; +} + +function qualifiedTableId(entry: KtxTableListEntry): string { + return `${entry.schema}.${entry.name}`; +} + +function tableTitle(entry: KtxTableListEntry): string { + return entry.kind === 'view' ? `${entry.name} (view)` : entry.name; +} + +function buildTreeInputs(discovered: readonly KtxTableListEntry[]): { + inputs: TreePickerNodeInput[]; + schemaIds: string[]; + allTables: string[]; +} { + const schemaSeen = new Set(); + const schemaIds: string[] = []; + for (const entry of discovered) { + if (!schemaSeen.has(entry.schema)) { + schemaSeen.add(entry.schema); + schemaIds.push(entry.schema); + } + } + const inputs: TreePickerNodeInput[] = []; + for (const schema of schemaIds) { + inputs.push({ id: schema, title: schema, archived: false, parentId: null }); + } + for (const entry of discovered) { + inputs.push({ + id: qualifiedTableId(entry), + title: tableTitle(entry), + archived: false, + parentId: entry.schema, + }); + } + return { inputs, schemaIds, allTables: discovered.map(qualifiedTableId) }; +} + +function initialSelectionForExisting( + existing: readonly string[], + byId: Map, +): string[] { + const tableIds = new Set( + [...byId.values()].filter((node) => node.parentId !== null).map((node) => node.id), + ); + const existingTables = new Set(existing.filter((id) => tableIds.has(id))); + const schemaChildren = new Map(); + for (const node of byId.values()) { + if (node.parentId === null && node.childIds.length > 0) { + schemaChildren.set(node.id, [...node.childIds]); + } + } + const result: string[] = []; + for (const [schema, children] of schemaChildren) { + const allChecked = children.length > 0 && children.every((childId) => existingTables.has(childId)); + if (allChecked) { + result.push(schema); + for (const childId of children) { + existingTables.delete(childId); + } + } + } + for (const id of existingTables) { + result.push(id); + } + return result; +} + +function initialSelectionFromDefaults( + defaultSchemas: readonly string[], + schemaIds: readonly string[], +): string[] { + const valid = new Set(schemaIds); + const filtered = defaultSchemas.filter((s) => valid.has(s)); + return filtered.length > 0 ? filtered : [...schemaIds]; +} + +function expandSelectedToTables( + selectedIds: readonly string[], + byId: Map, +): string[] { + const expanded: string[] = []; + const seen = new Set(); + for (const id of selectedIds) { + const node = byId.get(id); + if (!node) continue; + if (node.childIds.length === 0) { + if (node.parentId !== null && !seen.has(id)) { + seen.add(id); + expanded.push(id); + } + continue; + } + for (const childId of node.childIds) { + if (!seen.has(childId)) { + seen.add(childId); + expanded.push(childId); + } + } + } + return expanded; +} + +function schemasFromEnabledTables(enabledTables: readonly string[]): string[] { + const seen = new Set(); + const result: string[] = []; + for (const qualified of enabledTables) { + const schema = qualified.split('.')[0] ?? ''; + if (schema.length === 0 || seen.has(schema)) continue; + seen.add(schema); + result.push(schema); + } + return result; +} + +export async function pickDatabaseScope( + args: PickDatabaseScopeArgs, + io: KtxCliIo, + render: DatabaseTreePickerRenderer = defaultRenderer, +): Promise { + const { inputs, schemaIds, allTables } = buildTreeInputs(args.discovered); + const tree = buildPickerTree(inputs); + const byId = new Map(tree.map((node) => [node.id, node])); + const tableCount = allTables.length; + const schemaCount = schemaIds.length; + + const initialSelection = + args.existing.enabledTables.length > 0 + ? initialSelectionForExisting(args.existing.enabledTables, byId) + : initialSelectionFromDefaults(args.defaultSchemas, schemaIds); + + const initialState = buildInitialState({ + tree, + existingSelectedIds: initialSelection, + skipEmptyAction: 'save-empty', + }); + + const schemaWordPlural = schemaCount === 1 ? args.schemaNoun : args.schemaNounPlural; + const subtitleLines = [ + `Connection: ${args.connectionId}`, + `Found ${tableCount} ${tableCount === 1 ? 'table' : 'tables'} across ${schemaCount} ${schemaWordPlural}.`, + `Toggle a ${args.schemaNoun} to enable all of its tables, or expand to pick individual tables.`, + ]; + + const chrome: TreePickerChrome = { + title: `Choose tables to enable for ${args.connectionId}`, + subtitleLines, + skipEmptyMessage: + 'Nothing selected. Enable all tables? Press Enter to enable all or Escape to go back.', + }; + + const result = await render(chrome, initialState, io as TreePickerTuiIo); + if (result.kind === 'quit') { + return { kind: 'back' }; + } + + const enabledTables = + result.selectedIds.length === 0 ? allTables : expandSelectedToTables(result.selectedIds, byId); + const activeSchemas = args.supportsSchemaScope ? schemasFromEnabledTables(enabledTables) : []; + + return { kind: 'selected', activeSchemas, enabledTables }; +} diff --git a/packages/cli/src/doctor.test.ts b/packages/cli/src/doctor.test.ts index a7bf7714..c7199e9d 100644 --- a/packages/cli/src/doctor.test.ts +++ b/packages/cli/src/doctor.test.ts @@ -1,8 +1,7 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import type { KtxEmbeddingConfig, KtxEmbeddingHealthCheckOptions, KtxEmbeddingHealthCheckResult } from '@ktx/llm'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { formatDoctorReport, runKtxDoctor, @@ -31,53 +30,65 @@ function makeIo() { }; } -type EmbeddingHealthCheck = ( - config: KtxEmbeddingConfig, - options?: KtxEmbeddingHealthCheckOptions, -) => Promise; - -async function writeProjectConfig(projectDir: string, embeddingLines: string[]): Promise { - await writeFile( - join(projectDir, 'ktx.yaml'), - [ - 'project: warehouse', - 'connections:', - ' warehouse:', - ' driver: sqlite', - ' path: ./warehouse.db', - 'ingest:', - ' adapters:', - ' - live-database', - ' embeddings:', - ...embeddingLines.map((line) => ` ${line}`), - '', - ].join('\n'), - 'utf-8', - ); -} - describe('formatDoctorReport', () => { - it('prints exact fixes for failing setup checks', () => { + it('shows the failing check and its fix in plain output', () => { const checks: DoctorCheck[] = [ - { id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' }, + { id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127', group: 'toolchain' }, { id: 'native-sqlite', label: 'Native SQLite', status: 'fail', detail: 'Cannot load better-sqlite3', fix: 'Run: pnpm run native:rebuild', + group: 'toolchain', }, ]; - expect(formatDoctorReport({ title: 'KTX setup doctor', checks })).toBe( - [ - 'KTX setup doctor', - 'PASS Node 22+: v22.16.0 ABI 127', - 'FAIL Native SQLite: Cannot load better-sqlite3', - ' Fix: Run: pnpm run native:rebuild', - '', - ].join('\n'), - ); + const output = formatDoctorReport({ title: 'KTX status', checks }); + expect(output).toContain('KTX status'); + expect(output).toContain('✗ Environment'); + expect(output).toContain('1 of 2 need attention'); + expect(output).toContain('✗ Native SQLite: Cannot load better-sqlite3'); + expect(output).toContain('→ Run: pnpm run native:rebuild'); + expect(output).toContain('1 issue to fix.'); + }); + + it('lists what was checked when a group has all passing checks', () => { + const checks: DoctorCheck[] = [ + { id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0', group: 'toolchain' }, + { id: 'pnpm', label: 'pnpm 10.20+', status: 'pass', detail: '10.28.0', group: 'toolchain' }, + ]; + + const output = formatDoctorReport({ title: 'KTX status', checks }); + expect(output).toContain('✓ Environment'); + expect(output).toContain('Node 22+ · pnpm 10.20+'); + expect(output).not.toContain('v22.16.0'); + expect(output).toContain('Everything ready.'); + }); + + it('shows the underlying detail for a single-check group on the group line', () => { + const checks: DoctorCheck[] = [ + { + id: 'semantic-search-embeddings', + label: 'Semantic search embeddings', + status: 'pass', + detail: 'openai/text-embedding-3-small (1536d) probe succeeded', + group: 'search', + }, + ]; + + const output = formatDoctorReport({ title: 'KTX status', checks }); + expect(output).toContain('✓ Semantic search'); + expect(output).toContain('openai/text-embedding-3-small (1536d) probe succeeded'); + }); + + it('lists every check in verbose mode', () => { + const checks: DoctorCheck[] = [ + { id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0', group: 'toolchain' }, + ]; + + const output = formatDoctorReport({ title: 'KTX status', checks }, { verbose: true }); + expect(output).toContain('✓ Node 22+: v22.16.0'); }); }); @@ -127,6 +138,7 @@ describe('runSetupDoctorChecks', () => { status: 'fail', detail: 'pnpm not found', fix: 'Run: corepack enable && corepack prepare pnpm@10.28.0 --activate', + group: 'toolchain', }); expect(checks).toContainEqual({ id: 'package-build', @@ -134,6 +146,7 @@ describe('runSetupDoctorChecks', () => { status: 'fail', detail: 'Missing packages/cli/dist/bin.js', fix: 'Run: pnpm run build', + group: 'toolchain', }); }); @@ -154,9 +167,11 @@ describe('runSetupDoctorChecks', () => { const testIo = makeIo(); await expect( - runKtxDoctor({ command: 'setup', outputMode: 'plain', inputMode: 'disabled' }, testIo.io, { - runSetupChecks: async () => checks, - }), + runKtxDoctor( + { command: 'setup', outputMode: 'plain', inputMode: 'disabled', verbose: true }, + testIo.io, + { runSetupChecks: async () => checks }, + ), ).resolves.toBe(0); expect(checks).toContainEqual({ @@ -165,8 +180,9 @@ describe('runSetupDoctorChecks', () => { status: 'warn', detail: 'spawn corepack ENOENT', fix: 'Run: corepack enable', + group: 'toolchain', }); - expect(testIo.stdout()).toContain('WARN Corepack: spawn corepack ENOENT'); + expect(testIo.stdout()).toContain('⚠ Corepack: spawn corepack ENOENT'); expect(testIo.stderr()).toBe(''); }); }); @@ -204,12 +220,45 @@ describe('runKtxDoctor', () => { ), ).resolves.toBe(1); - expect(testIo.stdout()).toContain('KTX setup doctor'); - expect(testIo.stdout()).toContain('FAIL TypeScript package build: Missing packages/cli/dist/bin.js'); - expect(testIo.stdout()).toContain('Fix: Run: pnpm run build'); + expect(testIo.stdout()).toContain('KTX status'); + expect(testIo.stdout()).toContain('No project here yet.'); + expect(testIo.stdout()).toContain('Before you can run'); + expect(testIo.stdout()).toContain('✗ TypeScript package build: Missing packages/cli/dist/bin.js'); + expect(testIo.stdout()).toContain('→ Run: pnpm run build'); expect(testIo.stderr()).toBe(''); }); + it('leads with `ktx setup` and hides toolchain warnings when no project exists', async () => { + const testIo = makeIo(); + + await expect( + runKtxDoctor( + { command: 'setup', outputMode: 'plain', inputMode: 'disabled' }, + testIo.io, + { + runSetupChecks: async () => [ + { id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0', group: 'toolchain' }, + { + id: 'corepack', + label: 'Corepack', + status: 'warn', + detail: 'spawn corepack ENOENT', + fix: 'Run: corepack enable', + group: 'toolchain', + }, + ], + }, + ), + ).resolves.toBe(0); + + const out = testIo.stdout(); + expect(out).toContain('No project here yet.'); + expect(out).toContain('Run'); + expect(out).toContain('ktx setup'); + expect(out).not.toContain('Corepack'); + expect(out).not.toContain('Node 22+'); + }); + it('prints JSON setup report', async () => { const testIo = makeIo(); @@ -226,12 +275,13 @@ describe('runKtxDoctor', () => { ).resolves.toBe(0); expect(JSON.parse(testIo.stdout())).toEqual({ - title: 'KTX setup doctor', + title: 'KTX status', checks: [{ id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' }], }); }); it('runs project checks against a valid ktx.yaml', async () => { + process.env.ANTHROPIC_API_KEY = 'test-key'; await writeFile( join(tempDir, 'ktx.yaml'), [ @@ -240,83 +290,76 @@ describe('runKtxDoctor', () => { ' warehouse:', ' driver: sqlite', ' path: ./warehouse.db', + 'llm:', + ' provider:', + ' backend: anthropic', + ' models:', + ' default: claude-sonnet-4-5', 'ingest:', ' adapters:', ' - live-database', + ' embeddings:', + ' backend: openai', + ' model: text-embedding-3-small', + ' dimensions: 1536', '', ].join('\n'), 'utf-8', ); + process.env.OPENAI_API_KEY = 'test-key'; const testIo = makeIo(); await expect( runKtxDoctor( { command: 'project', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' }, testIo.io, - { - runSetupChecks: async () => [ - { id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' }, - ], - }, + {}, ), ).resolves.toBe(0); - expect(testIo.stdout()).toContain('KTX project doctor'); - expect(testIo.stdout()).toContain('PASS Project config: warehouse'); - expect(testIo.stdout()).toContain('PASS Connections: 1 configured'); + const out = testIo.stdout(); + expect(out).toContain('KTX status'); + expect(out).toContain('· warehouse'); + expect(out).toContain('Connections (1)'); + expect(out).toContain('LLM'); + expect(out).toContain('anthropic'); + expect(out).toContain('Embeddings'); + expect(out).toContain('Ready.'); + delete process.env.ANTHROPIC_API_KEY; + delete process.env.OPENAI_API_KEY; }); - it('includes Postgres historic-SQL readiness in project doctor output', async () => { + it('returns blocked verdict when LLM is not configured', async () => { await writeFile( join(tempDir, 'ktx.yaml'), [ 'project: warehouse', 'connections:', ' warehouse:', - ' driver: postgres', - ' url: env:WAREHOUSE_DATABASE_URL', - ' historicSql:', - ' enabled: true', - ' dialect: postgres', - 'ingest:', - ' adapters:', - ' - live-database', - ' - historic-sql', + ' driver: sqlite', + ' path: ./warehouse.db', '', ].join('\n'), 'utf-8', ); const testIo = makeIo(); - const runHistoricSqlDoctorChecks = vi.fn(async () => [ - { - id: 'historic-sql-postgres-warehouse', - label: 'Postgres Historic SQL (warehouse)', - status: 'pass' as const, - detail: - 'pg_stat_statements ready (PostgreSQL 16.4); info: pg_stat_statements.max is 1000; set it to at least 5000 to reduce query-template eviction churn', - }, - ]); await expect( runKtxDoctor( { command: 'project', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' }, testIo.io, - { - runSetupChecks: async () => [ - { id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' }, - ], - runHistoricSqlDoctorChecks, - }, + {}, ), - ).resolves.toBe(0); + ).resolves.toBe(1); - expect(runHistoricSqlDoctorChecks).toHaveBeenCalledTimes(1); - expect(testIo.stdout()).toContain('PASS Postgres Historic SQL (warehouse): pg_stat_statements ready'); - expect(testIo.stdout()).toContain('info: pg_stat_statements.max is 1000'); - expect(testIo.stdout()).not.toContain('Fix: Update the Postgres parameter group or config'); + expect(testIo.stdout()).toContain('no LLM configured'); + expect(testIo.stdout()).toContain('ktx setup'); }); it('warns about stale and unsupported per-driver connection fields', async () => { + process.env.ANTHROPIC_API_KEY = 'test-key'; + process.env.WAREHOUSE_DATABASE_URL = 'postgresql://reader@example.test/warehouse'; + process.env.NOTION_TOKEN = 'notion-secret'; await writeFile( join(tempDir, 'ktx.yaml'), [ @@ -342,6 +385,9 @@ describe('runKtxDoctor', () => { 'ingest:', ' adapters:', ' - live-database', + 'llm:', + ' provider:', + ' backend: anthropic', '', ].join('\n'), 'utf-8', @@ -352,166 +398,59 @@ describe('runKtxDoctor', () => { runKtxDoctor( { command: 'project', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' }, testIo.io, - { - runSetupChecks: async () => [ - { id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' }, - ], - runHistoricSqlDoctorChecks: async () => [], - }, + {}, ), ).resolves.toBe(0); - expect(testIo.stdout()).toContain('WARN Connection config (warehouse): connections.warehouse.readonly is no longer used.'); - expect(testIo.stdout()).toContain( - 'WARN Connection config (warehouse): connections.warehouse.historicSql.concurrency is no longer used.', - ); - expect(testIo.stdout()).toContain( - 'WARN Connection config (warehouse): connections.warehouse.historicSql.windowDays does not constrain pg_stat_statements.', - ); - expect(testIo.stdout()).toContain('WARN Connection config (local): connections.local.file_path was removed.'); - expect(testIo.stdout()).toContain( - 'WARN Connection config (docs): connections.docs.last_successful_cursor is local sync state.', - ); + const out = testIo.stdout(); + expect(out).toContain('Warnings'); + expect(out).toContain('connections.warehouse.readonly is no longer used.'); + expect(out).toContain('connections.warehouse.historicSql.concurrency is no longer used.'); + expect(out).toContain('connections.warehouse.historicSql.windowDays does not constrain pg_stat_statements.'); + expect(out).toContain('connections.local.file_path was removed.'); + expect(out).toContain('connections.docs.last_successful_cursor is local sync state.'); + delete process.env.ANTHROPIC_API_KEY; + delete process.env.WAREHOUSE_DATABASE_URL; + delete process.env.NOTION_TOKEN; }); it('warns when semantic-search embeddings are not configured', async () => { - await writeProjectConfig(tempDir, ['backend: deterministic', 'model: deterministic', 'dimensions: 8']); + process.env.ANTHROPIC_API_KEY = 'test-key'; + await writeFile( + join(tempDir, 'ktx.yaml'), + [ + 'project: warehouse', + 'connections:', + ' warehouse:', + ' driver: sqlite', + ' path: ./warehouse.db', + 'llm:', + ' provider:', + ' backend: anthropic', + 'ingest:', + ' adapters:', + ' - live-database', + ' embeddings:', + ' backend: deterministic', + ' model: deterministic', + ' dimensions: 8', + '', + ].join('\n'), + 'utf-8', + ); const testIo = makeIo(); await expect( runKtxDoctor( { command: 'project', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' }, testIo.io, - { - runSetupChecks: async () => [ - { id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' }, - ], - }, + {}, ), ).resolves.toBe(0); - expect(testIo.stdout()).toContain('WARN Semantic search embeddings: ingest.embeddings.backend is deterministic.'); - expect(testIo.stdout()).toContain( - 'Semantic lane will be skipped; lexical, dictionary, and token lanes remain available.', - ); - expect(testIo.stdout()).toContain( - `Fix: Run: ktx setup --project-dir ${tempDir} --no-input`, - ); - }); - - it('probes configured semantic-search embeddings for project doctor', async () => { - await writeProjectConfig(tempDir, [ - 'backend: sentence-transformers', - 'model: all-MiniLM-L6-v2', - 'dimensions: 384', - 'sentenceTransformers:', - ' base_url: http://127.0.0.1:8765', - " pathPrefix: ''", - ]); - const healthCheck = vi.fn(async () => ({ ok: true })); - const testIo = makeIo(); - - await expect( - runKtxDoctor( - { command: 'project', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' }, - testIo.io, - { - runSetupChecks: async () => [ - { id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' }, - ], - embeddingHealthCheck: healthCheck, - embeddingProbeTimeoutMs: 1234, - }, - ), - ).resolves.toBe(0); - - expect(healthCheck).toHaveBeenCalledWith( - { - backend: 'sentence-transformers', - model: 'all-MiniLM-L6-v2', - dimensions: 384, - sentenceTransformers: { baseURL: 'http://127.0.0.1:8765', pathPrefix: '' }, - }, - { text: 'KTX semantic search doctor probe', timeoutMs: 1234 }, - ); - expect(testIo.stdout()).toContain( - 'PASS Semantic search embeddings: sentence-transformers/all-MiniLM-L6-v2 (384d) probe succeeded', - ); - }); - - it('allows local sentence-transformers semantic-search probes enough time for cold start', async () => { - await writeProjectConfig(tempDir, [ - 'backend: sentence-transformers', - 'model: all-MiniLM-L6-v2', - 'dimensions: 384', - 'sentenceTransformers:', - ' base_url: http://127.0.0.1:8765', - " pathPrefix: ''", - ]); - const healthCheck = vi.fn(async () => ({ ok: true })); - const testIo = makeIo(); - - await expect( - runKtxDoctor( - { command: 'project', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' }, - testIo.io, - { - runSetupChecks: async () => [ - { id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' }, - ], - embeddingHealthCheck: healthCheck, - }, - ), - ).resolves.toBe(0); - - expect(healthCheck).toHaveBeenCalledWith( - expect.objectContaining({ - backend: 'sentence-transformers', - model: 'all-MiniLM-L6-v2', - dimensions: 384, - }), - { text: 'KTX semantic search doctor probe', timeoutMs: 120_000 }, - ); - }); - - it('reports unhealthy semantic-search embeddings as a warning in JSON output', async () => { - await writeProjectConfig(tempDir, [ - 'backend: sentence-transformers', - 'model: all-MiniLM-L6-v2', - 'dimensions: 384', - 'sentenceTransformers:', - ' base_url: http://127.0.0.1:8765', - " pathPrefix: ''", - ]); - const healthCheck = vi.fn(async () => ({ - ok: false, - message: 'connect ECONNREFUSED 127.0.0.1:8765', - })); - const testIo = makeIo(); - - await expect( - runKtxDoctor( - { command: 'project', projectDir: tempDir, outputMode: 'json', inputMode: 'disabled' }, - testIo.io, - { - runSetupChecks: async () => [ - { id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' }, - ], - embeddingHealthCheck: healthCheck, - }, - ), - ).resolves.toBe(0); - - const report = JSON.parse(testIo.stdout()) as { - checks: Array<{ id: string; label: string; status: string; detail: string; fix?: string }>; - }; - expect(report.checks).toContainEqual({ - id: 'semantic-search-embeddings', - label: 'Semantic search embeddings', - status: 'warn', - detail: - 'sentence-transformers/all-MiniLM-L6-v2 (384d) probe failed: connect ECONNREFUSED 127.0.0.1:8765. Semantic lane will be skipped; lexical, dictionary, and token lanes remain available.', - fix: `Run: ktx setup --project-dir ${tempDir} --no-input`, - }); + expect(testIo.stdout()).toContain('Embeddings'); + expect(testIo.stdout()).toContain('deterministic'); + expect(testIo.stdout()).toContain('semantic search degraded'); + delete process.env.ANTHROPIC_API_KEY; }); }); diff --git a/packages/cli/src/doctor.ts b/packages/cli/src/doctor.ts index 8279da90..9342f24e 100644 --- a/packages/cli/src/doctor.ts +++ b/packages/cli/src/doctor.ts @@ -4,15 +4,13 @@ import { access } from 'node:fs/promises'; import { join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { promisify } from 'node:util'; -import type { KtxLocalProject, KtxProjectEmbeddingConfig } from '@ktx/context/project'; -import type { KtxEmbeddingConfig, KtxEmbeddingHealthCheckOptions, KtxEmbeddingHealthCheckResult } from '@ktx/llm'; -import type { HistoricSqlDoctorDeps } from './historic-sql-doctor.js'; const execFileAsync = promisify(execFile); type DoctorStatus = 'pass' | 'warn' | 'fail'; type KtxDoctorOutputMode = 'plain' | 'json'; type KtxDoctorInputMode = 'auto' | 'disabled'; +type DoctorGroup = 'toolchain' | 'project' | 'search' | 'history'; export interface DoctorCheck { id: string; @@ -20,6 +18,7 @@ export interface DoctorCheck { status: DoctorStatus; detail: string; fix?: string; + group?: DoctorGroup; } interface DoctorReport { @@ -28,11 +27,22 @@ interface DoctorReport { } export type KtxDoctorArgs = - | { command: 'setup'; outputMode: KtxDoctorOutputMode; inputMode?: KtxDoctorInputMode } - | { command: 'project'; projectDir: string; outputMode: KtxDoctorOutputMode; inputMode?: KtxDoctorInputMode }; + | { + command: 'setup'; + outputMode: KtxDoctorOutputMode; + inputMode?: KtxDoctorInputMode; + verbose?: boolean; + } + | { + command: 'project'; + projectDir: string; + outputMode: KtxDoctorOutputMode; + inputMode?: KtxDoctorInputMode; + verbose?: boolean; + }; interface KtxDoctorIo { - stdout: { write(chunk: string): void }; + stdout: { isTTY?: boolean; write(chunk: string): void }; stderr: { write(chunk: string): void }; } @@ -44,20 +54,8 @@ interface SetupDoctorDeps { importBetterSqlite3?: () => Promise; } -type EmbeddingHealthCheck = ( - config: KtxEmbeddingConfig, - options?: KtxEmbeddingHealthCheckOptions, -) => Promise; - -interface SemanticSearchDoctorDeps { - env?: NodeJS.ProcessEnv; - embeddingHealthCheck?: EmbeddingHealthCheck; - embeddingProbeTimeoutMs?: number; -} - -interface KtxDoctorDeps extends SemanticSearchDoctorDeps, HistoricSqlDoctorDeps { +interface KtxDoctorDeps { runSetupChecks?: () => Promise; - runHistoricSqlDoctorChecks?: (project: KtxLocalProject, deps: HistoricSqlDoctorDeps) => Promise; } function workspaceRootDir(): string { @@ -118,197 +116,6 @@ function check(status: DoctorStatus, id: string, label: string, detail: string, return fix ? { id, label, status, detail, fix } : { id, label, status, detail }; } -interface ConnectionConfigWarning { - id: string; - connectionId: string; - detail: string; - fix: string; -} - -function isRecord(value: unknown): value is Record { - return typeof value === 'object' && value !== null && !Array.isArray(value); -} - -function hasOwnField(value: Record, key: string): boolean { - return Object.prototype.hasOwnProperty.call(value, key); -} - -function connectionConfigWarning( - connectionId: string, - key: string, - detail: string, - fix: string, -): ConnectionConfigWarning { - return { - id: `connection-config-${connectionId}-${key}`.replace(/[^a-zA-Z0-9_-]/g, '-'), - connectionId, - detail, - fix, - }; -} - -function connectionConfigWarnings(project: KtxLocalProject): ConnectionConfigWarning[] { - const warnings: ConnectionConfigWarning[] = []; - for (const [connectionId, connection] of Object.entries(project.config.connections)) { - const driver = String(connection.driver ?? '').toLowerCase(); - if (hasOwnField(connection, 'readonly')) { - warnings.push( - connectionConfigWarning( - connectionId, - 'readonly', - `connections.${connectionId}.readonly is no longer used.`, - `Remove connections.${connectionId}.readonly from ktx.yaml.`, - ), - ); - } - - if ((driver === 'sqlite' || driver === 'sqlite3') && hasOwnField(connection, 'file_path')) { - warnings.push( - connectionConfigWarning( - connectionId, - 'file-path', - `connections.${connectionId}.file_path was removed.`, - `Rename connections.${connectionId}.file_path to path.`, - ), - ); - } - - if (driver === 'notion' && hasOwnField(connection, 'last_successful_cursor')) { - warnings.push( - connectionConfigWarning( - connectionId, - 'last-successful-cursor', - `connections.${connectionId}.last_successful_cursor is local sync state.`, - 'Remove it from ktx.yaml. KTX stores the Notion cursor in .ktx/db.sqlite.', - ), - ); - } - - const historicSql = isRecord(connection.historicSql) ? connection.historicSql : null; - if (!historicSql) { - continue; - } - if (hasOwnField(historicSql, 'concurrency')) { - warnings.push( - connectionConfigWarning( - connectionId, - 'historic-sql-concurrency', - `connections.${connectionId}.historicSql.concurrency is no longer used.`, - `Remove connections.${connectionId}.historicSql.concurrency from ktx.yaml.`, - ), - ); - } - const historicDialect = String(historicSql.dialect ?? driver).toLowerCase(); - if ( - (historicDialect === 'postgres' || historicDialect === 'postgresql') && - hasOwnField(historicSql, 'windowDays') - ) { - warnings.push( - connectionConfigWarning( - connectionId, - 'historic-sql-window-days', - `connections.${connectionId}.historicSql.windowDays does not constrain pg_stat_statements.`, - `Remove connections.${connectionId}.historicSql.windowDays from ktx.yaml.`, - ), - ); - } - } - return warnings; -} - -const SEMANTIC_SEARCH_HEALTH_TEXT = 'KTX semantic search doctor probe'; -const SEMANTIC_SEARCH_HEALTH_TIMEOUT_MS = 5_000; -const SEMANTIC_SEARCH_LOCAL_HEALTH_TIMEOUT_MS = 120_000; - -function semanticEmbeddingSetupFix(projectDir: string, backend: KtxProjectEmbeddingConfig['backend']): string { - if (backend === 'openai') { - return `Set OPENAI_API_KEY or rerun: ktx setup --project-dir ${projectDir} --embedding-backend openai --no-input`; - } - return `Run: ktx setup --project-dir ${projectDir} --no-input`; -} - -function embeddingConfigLabel(config: KtxProjectEmbeddingConfig | KtxEmbeddingConfig): string { - const model = config.model?.trim() || 'model not configured'; - return `${config.backend}/${model} (${config.dimensions}d)`; -} - -function semanticLaneFallbackDetail(reason: string): string { - return `${reason}. Semantic lane will be skipped; lexical, dictionary, and token lanes remain available.`; -} - -async function defaultEmbeddingHealthCheck( - config: KtxEmbeddingConfig, - options?: KtxEmbeddingHealthCheckOptions, -): Promise { - const { runKtxEmbeddingHealthCheck } = await import('@ktx/llm'); - return runKtxEmbeddingHealthCheck(config, options); -} - -async function runSemanticSearchEmbeddingCheck( - config: KtxProjectEmbeddingConfig, - projectDir: string, - deps: SemanticSearchDoctorDeps = {}, -): Promise { - if (config.backend === 'none' || config.backend === 'deterministic') { - return check( - 'warn', - 'semantic-search-embeddings', - 'Semantic search embeddings', - semanticLaneFallbackDetail(`ingest.embeddings.backend is ${config.backend}`), - semanticEmbeddingSetupFix(projectDir, config.backend), - ); - } - - try { - const { resolveLocalKtxEmbeddingConfig } = await import('@ktx/context'); - const resolved = resolveLocalKtxEmbeddingConfig(config, deps.env ?? process.env); - if (!resolved) { - return check( - 'warn', - 'semantic-search-embeddings', - 'Semantic search embeddings', - semanticLaneFallbackDetail(`No runtime embedding config resolved for ${embeddingConfigLabel(config)}`), - semanticEmbeddingSetupFix(projectDir, config.backend), - ); - } - - const healthCheck = deps.embeddingHealthCheck ?? defaultEmbeddingHealthCheck; - const timeoutMs = - deps.embeddingProbeTimeoutMs ?? - (resolved.backend === 'sentence-transformers' - ? SEMANTIC_SEARCH_LOCAL_HEALTH_TIMEOUT_MS - : SEMANTIC_SEARCH_HEALTH_TIMEOUT_MS); - const health = await healthCheck(resolved, { - text: SEMANTIC_SEARCH_HEALTH_TEXT, - timeoutMs, - }); - if (health.ok) { - return check( - 'pass', - 'semantic-search-embeddings', - 'Semantic search embeddings', - `${embeddingConfigLabel(resolved)} probe succeeded`, - ); - } - - return check( - 'warn', - 'semantic-search-embeddings', - 'Semantic search embeddings', - semanticLaneFallbackDetail(`${embeddingConfigLabel(resolved)} probe failed: ${health.message}`), - semanticEmbeddingSetupFix(projectDir, config.backend), - ); - } catch (error) { - return check( - 'warn', - 'semantic-search-embeddings', - 'Semantic search embeddings', - semanticLaneFallbackDetail(`${embeddingConfigLabel(config)} probe failed: ${failureMessage(error)}`), - semanticEmbeddingSetupFix(projectDir, config.backend), - ); - } -} - export async function runSetupDoctorChecks(deps: SetupDoctorDeps = {}): Promise { const env = deps.env ?? process.env; const root = deps.workspaceRoot ?? workspaceRootDir(); @@ -402,67 +209,231 @@ export async function runSetupDoctorChecks(deps: SetupDoctorDeps = {}): Promise< ); } - return checks; + return checks.map((entry) => ({ ...entry, group: 'toolchain' })); } -async function runProjectChecks(projectDir: string, deps: KtxDoctorDeps = {}): Promise { - const { loadKtxProject } = await import('@ktx/context/project'); - const checks: DoctorCheck[] = []; - try { - const project = await loadKtxProject({ projectDir }); - checks.push(check('pass', 'project-config', 'Project config', project.config.project)); - const connectionCount = Object.keys(project.config.connections).length; - checks.push( - connectionCount > 0 - ? check('pass', 'connections', 'Connections', `${connectionCount} configured`) - : check( - 'warn', - 'connections', - 'Connections', - '0 configured', - 'Add a connection to ktx.yaml or run `ktx setup`', - ), - ); - for (const warning of connectionConfigWarnings(project)) { - checks.push( - check( - 'warn', - warning.id, - `Connection config (${warning.connectionId})`, - warning.detail, - warning.fix, - ), - ); - } - checks.push(check('pass', 'storage', 'Storage', `${project.config.storage.state}/${project.config.storage.search}`)); - checks.push(check('pass', 'llm-provider', 'LLM provider', project.config.llm.provider.backend)); - checks.push(await runSemanticSearchEmbeddingCheck(project.config.ingest.embeddings, projectDir, deps)); - const runHistoricSqlDoctorChecks = - deps.runHistoricSqlDoctorChecks ?? (await import('./historic-sql-doctor.js')).runPostgresHistoricSqlDoctorChecks; - checks.push(...(await runHistoricSqlDoctorChecks(project, deps))); - } catch (error) { - checks.push( - check( - 'fail', - 'project-config', - 'Project config', - failureMessage(error), - `Run: ktx init ${projectDir} --name `, - ), - ); +const STATUS_SYMBOL: Record = { pass: '✓', warn: '⚠', fail: '✗' }; + +const GROUP_ORDER: DoctorGroup[] = ['toolchain', 'project', 'search', 'history']; + +const GROUP_LABEL: Record = { + toolchain: 'Environment', + project: 'Project', + search: 'Semantic search', + history: 'Query history', +}; + +function shouldUseColor(io: KtxDoctorIo): boolean { + if (io.stdout.isTTY !== true) return false; + const env = process.env; + return !env.NO_COLOR && env.TERM !== 'dumb' && !env.CI; +} + +function styleStatus(useColor: boolean, status: DoctorStatus, text: string): string { + if (!useColor) return text; + const code = status === 'pass' ? 32 : status === 'warn' ? 33 : 31; + return `\u001b[${code}m${text}\u001b[39m`; +} + +function styleDim(useColor: boolean, text: string): string { + return useColor ? `\u001b[2m${text}\u001b[22m` : text; +} + +function styleBold(useColor: boolean, text: string): string { + return useColor ? `\u001b[1m${text}\u001b[22m` : text; +} + +function groupOf(entry: DoctorCheck): DoctorGroup { + return entry.group ?? 'project'; +} + +function aggregateStatus(checks: DoctorCheck[]): DoctorStatus { + if (checks.some((c) => c.status === 'fail')) return 'fail'; + if (checks.some((c) => c.status === 'warn')) return 'warn'; + return 'pass'; +} + +function abbreviateHome(filePath: string | undefined): string | undefined { + if (!filePath) return filePath; + const home = process.env.HOME; + if (home && (filePath === home || filePath.startsWith(`${home}/`))) { + return filePath === home ? '~' : `~${filePath.slice(home.length)}`; } - return checks; + return filePath; } -export function formatDoctorReport(report: DoctorReport): string { - const lines = [report.title]; - for (const item of report.checks) { - lines.push(`${item.status.toUpperCase()} ${item.label}: ${item.detail}`); - if (item.fix) { - lines.push(` Fix: ${item.fix}`); +function groupSummaryWhenAllPass(entries: DoctorCheck[]): string { + if (entries.length === 1) { + const only = entries[0]!; + return only.detail || only.label; + } + return entries.map((c) => c.label).join(' · '); +} + +interface RenderOptions { + verbose: boolean; + useColor: boolean; + durationMs?: number; + projectName?: string; + projectDir?: string; + command?: 'setup' | 'project'; +} + +const NEXT_STEPS_PROJECT = ['ktx scan', 'ktx wiki', 'ktx sl ask "…"']; + +export function formatDoctorReport(report: DoctorReport, options: Partial = {}): string { + const opts: RenderOptions = { + verbose: options.verbose ?? false, + useColor: options.useColor ?? false, + durationMs: options.durationMs, + projectName: options.projectName, + projectDir: options.projectDir, + command: options.command, + }; + return renderPlainReport(report, opts); +} + +function renderSetupReport(report: DoctorReport, options: RenderOptions): string { + const { verbose, useColor } = options; + const dim = (text: string) => styleDim(useColor, text); + const bold = (text: string) => styleBold(useColor, text); + const status = (s: DoctorStatus, text: string) => styleStatus(useColor, s, text); + const symbol = (s: DoctorStatus) => status(s, STATUS_SYMBOL[s]); + + const fails = report.checks.filter((c) => c.status === 'fail'); + const lines: string[] = []; + lines.push(bold(report.title)); + lines.push(''); + lines.push(` No project here yet.`); + lines.push(''); + + if (fails.length > 0) { + lines.push(` Before you can run ${bold('ktx setup')}, fix this:`); + for (const entry of fails) { + lines.push(` ${symbol('fail')} ${entry.label}: ${entry.detail}`); + if (entry.fix) { + lines.push(` ${dim(`→ ${entry.fix}`)}`); + } } + lines.push(''); + } else { + lines.push(` Run ${bold('ktx setup')} to get started.`); + lines.push(''); + } + + if (verbose) { + lines.push(dim(' Toolchain:')); + for (const entry of report.checks) { + lines.push(` ${symbol(entry.status)} ${entry.label}: ${entry.detail}`); + if (entry.fix && entry.status !== 'pass') { + lines.push(` ${dim(`→ ${entry.fix}`)}`); + } + } + lines.push(''); + } + + return lines.join('\n'); +} + +function renderPlainReport(report: DoctorReport, options: RenderOptions): string { + if (options.command === 'setup') return renderSetupReport(report, options); + const { verbose, useColor, durationMs, projectName, projectDir } = options; + const dim = (text: string) => styleDim(useColor, text); + const bold = (text: string) => styleBold(useColor, text); + const status = (s: DoctorStatus, text: string) => styleStatus(useColor, s, text); + const symbol = (s: DoctorStatus) => status(s, STATUS_SYMBOL[s]); + + const lines: string[] = []; + const titleParts: string[] = [bold(report.title)]; + if (projectName) titleParts.push(projectName); + const abbreviatedDir = abbreviateHome(projectDir); + const titleLine = titleParts.join(` ${dim('·')} `); + const dirSuffix = abbreviatedDir ? ` ${dim(`(${abbreviatedDir})`)}` : ''; + lines.push(`${titleLine}${dirSuffix}`); + lines.push(''); + + const groups = new Map(); + for (const entry of report.checks) { + const group = groupOf(entry); + const bucket = groups.get(group) ?? []; + bucket.push(entry); + groups.set(group, bucket); + } + + const orderedGroups: DoctorGroup[] = []; + for (const g of GROUP_ORDER) { + if (groups.has(g)) orderedGroups.push(g); + } + for (const g of groups.keys()) { + if (!orderedGroups.includes(g)) orderedGroups.push(g); + } + + const labelWidth = orderedGroups.reduce( + (max, g) => Math.max(max, (GROUP_LABEL[g] ?? g).length), + 0, + ); + + for (const group of orderedGroups) { + const entries = groups.get(group) ?? []; + const head = aggregateStatus(entries); + const nonPass = entries.filter((c) => c.status !== 'pass'); + const label = (GROUP_LABEL[group] ?? group).padEnd(labelWidth); + + if (nonPass.length === 0) { + lines.push(` ${symbol(head)} ${label} ${dim(groupSummaryWhenAllPass(entries))}`); + if (verbose) { + for (const entry of entries) { + lines.push(` ${symbol(entry.status)} ${entry.label}: ${entry.detail}`); + } + } + continue; + } + + if (entries.length === 1) { + const only = entries[0]!; + lines.push(` ${symbol(only.status)} ${label} ${only.detail}`); + if (only.fix) { + lines.push(` ${' '.repeat(2 + labelWidth + 4)}${dim(`→ ${only.fix}`)}`); + } + continue; + } + + lines.push(` ${symbol(head)} ${label} ${dim(`${nonPass.length} of ${entries.length} need attention`)}`); + for (const entry of entries) { + if (entry.status === 'pass' && !verbose) continue; + lines.push(` ${symbol(entry.status)} ${entry.label}: ${entry.detail}`); + if (entry.fix) { + lines.push(` ${dim(`→ ${entry.fix}`)}`); + } + } + } + + lines.push(''); + + const totalFail = report.checks.filter((c) => c.status === 'fail').length; + const totalWarn = report.checks.filter((c) => c.status === 'warn').length; + const durationText = durationMs !== undefined ? ` ${dim(`(${(durationMs / 1000).toFixed(2)}s)`)}` : ''; + + if (totalFail === 0 && totalWarn === 0) { + const hint = ` ${dim('Try:')} ${NEXT_STEPS_PROJECT.join(dim(' · '))}`; + lines.push(`${status('pass', 'Everything ready.')}${hint}${durationText}`); + } else if (totalFail === 0) { + const word = totalWarn === 1 ? 'warning' : 'warnings'; + lines.push( + `${status('warn', `${totalWarn} ${word}.`)} ${dim('Run')} ktx status --verbose ${dim('for full details.')}${durationText}`, + ); + } else { + const fWord = totalFail === 1 ? 'issue' : 'issues'; + const warnSuffix = + totalWarn > 0 + ? ` ${dim('·')} ${status('warn', `${totalWarn} ${totalWarn === 1 ? 'warning' : 'warnings'}`)}` + : ''; + lines.push( + `${status('fail', `${totalFail} ${fWord} to fix.`)}${warnSuffix}${durationText}`, + ); } lines.push(''); + return lines.join('\n'); } @@ -470,12 +441,12 @@ function hasFailures(report: DoctorReport): boolean { return report.checks.some((item) => item.status === 'fail'); } -function writeReport(report: DoctorReport, outputMode: KtxDoctorOutputMode, io: KtxDoctorIo): void { +function writeReport(report: DoctorReport, outputMode: KtxDoctorOutputMode, io: KtxDoctorIo, options: RenderOptions): void { if (outputMode === 'json') { io.stdout.write(`${JSON.stringify(report, null, 2)}\n`); return; } - io.stdout.write(formatDoctorReport(report)); + io.stdout.write(renderPlainReport(report, options)); } export async function runKtxDoctor( @@ -483,18 +454,41 @@ export async function runKtxDoctor( io: KtxDoctorIo = process, deps: KtxDoctorDeps = {}, ): Promise { + const startedAt = Date.now(); try { const runSetupChecks = deps.runSetupChecks ?? (() => runSetupDoctorChecks()); - const setupChecks = await runSetupChecks(); - const report: DoctorReport = - args.command === 'setup' - ? { title: 'KTX setup doctor', checks: setupChecks } - : { - title: 'KTX project doctor', - checks: [...setupChecks, ...(await runProjectChecks(args.projectDir, deps))], - }; - writeReport(report, args.outputMode, io); + if (args.command === 'project') { + const { loadKtxProject } = await import('@ktx/context/project'); + const { buildProjectStatus, renderProjectStatus } = await import('./status-project.js'); + const project = await loadKtxProject({ projectDir: args.projectDir }); + const projectStatus = buildProjectStatus(project); + const verbose = args.verbose ?? false; + const toolchainChecks = verbose ? await runSetupChecks() : undefined; + if (args.outputMode === 'json') { + io.stdout.write(`${JSON.stringify(projectStatus, null, 2)}\n`); + } else { + io.stdout.write( + renderProjectStatus(projectStatus, { + verbose, + useColor: shouldUseColor(io), + durationMs: Date.now() - startedAt, + toolchainChecks, + }), + ); + } + return projectStatus.verdict === 'blocked' ? 1 : 0; + } + + const setupChecks = await runSetupChecks(); + const report: DoctorReport = { title: 'KTX status', checks: setupChecks }; + const renderOptions: RenderOptions = { + verbose: args.verbose ?? false, + useColor: shouldUseColor(io), + durationMs: Date.now() - startedAt, + command: args.command, + }; + writeReport(report, args.outputMode, io, renderOptions); return hasFailures(report) ? 1 : 0; } catch (error) { io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); diff --git a/packages/cli/src/historic-sql-doctor.test.ts b/packages/cli/src/historic-sql-doctor.test.ts deleted file mode 100644 index f3bc347e..00000000 --- a/packages/cli/src/historic-sql-doctor.test.ts +++ /dev/null @@ -1,202 +0,0 @@ -import { buildDefaultKtxProjectConfig, type KtxProjectConnectionConfig } from '@ktx/context/project'; -import { HistoricSqlExtensionMissingError } from '@ktx/context/ingest'; -import { describe, expect, it, vi } from 'vitest'; -import { - runPostgresHistoricSqlDoctorChecks, - type HistoricSqlDoctorProject, - type PostgresHistoricSqlDoctorProbe, -} from './historic-sql-doctor.js'; - -function projectWithConnections(connections: Record): HistoricSqlDoctorProject { - return { - projectDir: '/tmp/ktx-project', - config: { - ...buildDefaultKtxProjectConfig('warehouse'), - connections, - ingest: { - ...buildDefaultKtxProjectConfig('warehouse').ingest, - adapters: ['live-database', 'historic-sql'], - }, - }, - }; -} - -describe('runPostgresHistoricSqlDoctorChecks', () => { - it('passes when no Postgres historic-SQL connections are enabled', async () => { - const checks = await runPostgresHistoricSqlDoctorChecks( - projectWithConnections({ - warehouse: { driver: 'sqlite', path: './warehouse.db' }, - }), - { - postgresHistoricSqlProbe: vi.fn(), - }, - ); - - expect(checks).toEqual([ - { - id: 'historic-sql-postgres', - label: 'Postgres Historic SQL', - status: 'pass', - detail: 'No enabled Postgres historic-SQL connections', - }, - ]); - }); - - it('passes when the PGSS probe succeeds without warnings', async () => { - const probe = vi.fn(async () => ({ - pgServerVersion: 'PostgreSQL 16.4', - warnings: [], - })); - - const checks = await runPostgresHistoricSqlDoctorChecks( - projectWithConnections({ - warehouse: { - driver: 'postgres', - url: 'env:WAREHOUSE_DATABASE_URL', - historicSql: { enabled: true, dialect: 'postgres' }, - }, - }), - { postgresHistoricSqlProbe: probe }, - ); - - expect(probe).toHaveBeenCalledWith({ - projectDir: '/tmp/ktx-project', - connectionId: 'warehouse', - connection: { - driver: 'postgres', - url: 'env:WAREHOUSE_DATABASE_URL', - historicSql: { enabled: true, dialect: 'postgres' }, - }, - env: process.env, - }); - expect(checks).toEqual([ - { - id: 'historic-sql-postgres-warehouse', - label: 'Postgres Historic SQL (warehouse)', - status: 'pass', - detail: 'pg_stat_statements ready (PostgreSQL 16.4)', - }, - ]); - }); - - it('passes with an informational note when only pg_stat_statements.max is below the recommended floor', async () => { - const checks = await runPostgresHistoricSqlDoctorChecks( - projectWithConnections({ - warehouse: { - driver: 'postgres', - url: 'env:WAREHOUSE_DATABASE_URL', - historicSql: { enabled: true, dialect: 'postgres' }, - }, - }), - { - postgresHistoricSqlProbe: async () => ({ - pgServerVersion: 'PostgreSQL 16.4', - warnings: [], - info: [ - 'pg_stat_statements.max is 1000; set it to at least 5000 to reduce query-template eviction churn', - ], - }), - }, - ); - - expect(checks).toEqual([ - { - id: 'historic-sql-postgres-warehouse', - label: 'Postgres Historic SQL (warehouse)', - status: 'pass', - detail: - 'pg_stat_statements ready (PostgreSQL 16.4); info: pg_stat_statements.max is 1000; set it to at least 5000 to reduce query-template eviction churn', - }, - ]); - }); - - it('warns when pg_stat_statements tracking is disabled', async () => { - const checks = await runPostgresHistoricSqlDoctorChecks( - projectWithConnections({ - warehouse: { - driver: 'postgres', - url: 'env:WAREHOUSE_DATABASE_URL', - historicSql: { enabled: true, dialect: 'postgres' }, - }, - }), - { - postgresHistoricSqlProbe: async () => ({ - pgServerVersion: 'PostgreSQL 16.4', - warnings: [ - 'pg_stat_statements.track is none; set it to top or all in the Postgres parameter group or config', - ], - info: [ - 'pg_stat_statements.max is 1000; set it to at least 5000 to reduce query-template eviction churn', - ], - }), - }, - ); - - expect(checks).toEqual([ - { - id: 'historic-sql-postgres-warehouse', - label: 'Postgres Historic SQL (warehouse)', - status: 'warn', - detail: - 'pg_stat_statements ready (PostgreSQL 16.4) with warnings: pg_stat_statements.track is none; set it to top or all in the Postgres parameter group or config; info: pg_stat_statements.max is 1000; set it to at least 5000 to reduce query-template eviction churn', - fix: 'Update the Postgres parameter group or config, then rerun `ktx status --project-dir /tmp/ktx-project`', - }, - ]); - }); - - it('fails when a connection has postgres historic SQL but is not a Postgres driver', async () => { - const checks = await runPostgresHistoricSqlDoctorChecks( - projectWithConnections({ - warehouse: { - driver: 'mysql', - url: 'env:WAREHOUSE_DATABASE_URL', - historicSql: { enabled: true, dialect: 'postgres' }, - }, - }), - { - postgresHistoricSqlProbe: vi.fn(), - }, - ); - - expect(checks).toEqual([ - { - id: 'historic-sql-postgres-warehouse', - label: 'Postgres Historic SQL (warehouse)', - status: 'fail', - detail: 'connections.warehouse.historicSql.dialect is postgres but driver is mysql', - fix: 'Set connections.warehouse.driver to postgres or disable historicSql for this connection', - }, - ]); - }); - - it('maps PGSS capability errors to actionable failures', async () => { - const checks = await runPostgresHistoricSqlDoctorChecks( - projectWithConnections({ - warehouse: { - driver: 'postgres', - url: 'env:WAREHOUSE_DATABASE_URL', - historicSql: { enabled: true, dialect: 'postgres' }, - }, - }), - { - postgresHistoricSqlProbe: async () => { - throw new HistoricSqlExtensionMissingError({ - dialect: 'postgres', - message: 'pg_stat_statements extension is not installed in the connection database.', - remediation: 'Run CREATE EXTENSION pg_stat_statements; against the connection database.', - }); - }, - }, - ); - - expect(checks).toEqual([ - { - id: 'historic-sql-postgres-warehouse', - label: 'Postgres Historic SQL (warehouse)', - status: 'fail', - detail: 'pg_stat_statements extension is not installed in the connection database.', - fix: 'Run CREATE EXTENSION pg_stat_statements; against the connection database.', - }, - ]); - }); -}); diff --git a/packages/cli/src/historic-sql-doctor.ts b/packages/cli/src/historic-sql-doctor.ts deleted file mode 100644 index bb9a681c..00000000 --- a/packages/cli/src/historic-sql-doctor.ts +++ /dev/null @@ -1,167 +0,0 @@ -import type { KtxProjectConfig, KtxProjectConnectionConfig } from '@ktx/context/project'; -import type { DoctorCheck } from './doctor.js'; - -export interface HistoricSqlDoctorProject { - projectDir: string; - config: Pick; -} - -export interface PostgresHistoricSqlDoctorProbeInput { - projectDir: string; - connectionId: string; - connection: KtxProjectConnectionConfig; - env: NodeJS.ProcessEnv; -} - -export interface PostgresHistoricSqlDoctorProbeResult { - pgServerVersion: string; - warnings: string[]; - info?: string[]; -} - -export type PostgresHistoricSqlDoctorProbe = ( - input: PostgresHistoricSqlDoctorProbeInput, -) => Promise; - -export interface HistoricSqlDoctorDeps { - env?: NodeJS.ProcessEnv; - postgresHistoricSqlProbe?: PostgresHistoricSqlDoctorProbe; -} - -function check(status: DoctorCheck['status'], id: string, label: string, detail: string, fix?: string): DoctorCheck { - return fix ? { id, label, status, detail, fix } : { id, label, status, detail }; -} - -function historicSqlRecord(connection: KtxProjectConnectionConfig): Record | null { - const historicSql = connection.historicSql; - return historicSql && typeof historicSql === 'object' && !Array.isArray(historicSql) - ? (historicSql as Record) - : null; -} - -function isEnabledPostgresHistoricSql(connection: KtxProjectConnectionConfig): boolean { - const historicSql = historicSqlRecord(connection); - return historicSql?.enabled === true && historicSql.dialect === 'postgres'; -} - -function isPostgresDriver(connection: KtxProjectConnectionConfig): boolean { - const driver = String(connection.driver ?? '').toLowerCase(); - return driver === 'postgres' || driver === 'postgresql'; -} - -function checkId(connectionId: string): string { - return `historic-sql-postgres-${connectionId.replace(/[^a-z0-9_-]+/gi, '-')}`; -} - -function capabilityFailureFix(error: unknown, connectionId: string, projectDir: string): string { - if (error instanceof Error && error.name === 'HistoricSqlExtensionMissingError' && 'remediation' in error) { - return String(error.remediation); - } - if (error instanceof Error && error.name === 'HistoricSqlGrantsMissingError' && 'remediation' in error) { - return String(error.remediation); - } - if (error instanceof Error && error.name === 'HistoricSqlVersionUnsupportedError') { - return 'Use PostgreSQL 14 or newer, or disable historicSql for this connection'; - } - return `Fix connections.${connectionId} Postgres settings, then rerun \`ktx status --project-dir ${projectDir}\``; -} - -function failureDetail(error: unknown): string { - if (error instanceof Error && error.message.trim().length > 0) { - return error.message.trim().split('\n')[0] ?? error.message.trim(); - } - return String(error); -} - -function readinessDetail(result: PostgresHistoricSqlDoctorProbeResult): string { - const warningText = result.warnings.length > 0 ? ` with warnings: ${result.warnings.join('; ')}` : ''; - const info = result.info ?? []; - const infoText = info.length > 0 ? `; info: ${info.join('; ')}` : ''; - return `pg_stat_statements ready (${result.pgServerVersion})${warningText}${infoText}`; -} - -async function defaultPostgresHistoricSqlProbe( - input: PostgresHistoricSqlDoctorProbeInput, -): Promise { - const [{ PostgresPgssReader }, { KtxPostgresHistoricSqlQueryClient, isKtxPostgresConnectionConfig }] = - await Promise.all([import('@ktx/context/ingest'), import('@ktx/connector-postgres')]); - - const inputDriver = input.connection.driver ?? 'unknown'; - if (!isKtxPostgresConnectionConfig(input.connection)) { - throw new Error(`Native PostgreSQL connector cannot run driver "${inputDriver}"`); - } - - const client = new KtxPostgresHistoricSqlQueryClient({ - connectionId: input.connectionId, - connection: input.connection, - env: input.env, - }); - try { - return await new PostgresPgssReader().probe(client); - } finally { - await client.cleanup(); - } -} - -export async function runPostgresHistoricSqlDoctorChecks( - project: HistoricSqlDoctorProject, - deps: HistoricSqlDoctorDeps = {}, -): Promise { - const targets = Object.entries(project.config.connections) - .filter(([, connection]) => isEnabledPostgresHistoricSql(connection)) - .sort(([left], [right]) => left.localeCompare(right)); - - if (targets.length === 0) { - return [ - check('pass', 'historic-sql-postgres', 'Postgres Historic SQL', 'No enabled Postgres historic-SQL connections'), - ]; - } - - const probe = deps.postgresHistoricSqlProbe ?? defaultPostgresHistoricSqlProbe; - const env = deps.env ?? process.env; - const checks: DoctorCheck[] = []; - for (const [connectionId, connection] of targets) { - const label = `Postgres Historic SQL (${connectionId})`; - if (!isPostgresDriver(connection)) { - checks.push( - check( - 'fail', - checkId(connectionId), - label, - `connections.${connectionId}.historicSql.dialect is postgres but driver is ${String(connection.driver)}`, - `Set connections.${connectionId}.driver to postgres or disable historicSql for this connection`, - ), - ); - continue; - } - - try { - const result = await probe({ projectDir: project.projectDir, connectionId, connection, env }); - if (result.warnings.length > 0) { - checks.push( - check( - 'warn', - checkId(connectionId), - label, - readinessDetail(result), - `Update the Postgres parameter group or config, then rerun \`ktx status --project-dir ${project.projectDir}\``, - ), - ); - } else { - checks.push(check('pass', checkId(connectionId), label, readinessDetail(result))); - } - } catch (error) { - checks.push( - check( - 'fail', - checkId(connectionId), - label, - failureDetail(error), - capabilityFailureFix(error, connectionId, project.projectDir), - ), - ); - } - } - - return checks; -} diff --git a/packages/cli/src/index.test.ts b/packages/cli/src/index.test.ts index cd635d78..d1c2587e 100644 --- a/packages/cli/src/index.test.ts +++ b/packages/cli/src/index.test.ts @@ -444,20 +444,54 @@ describe('runKtxCli', () => { expect(io.stderr()).toContain('Choose only one runtime install mode: --yes or --no-input'); }); - it('documents setup as a bare command without subcommands', async () => { + it('documents setup with only the common interactive options visible', async () => { const testIo = makeIo(); await expect(runKtxCli(['setup', '--help'], testIo.io)).resolves.toBe(0); - expect(testIo.stdout()).toContain('Usage: ktx setup [options]'); - expect(testIo.stdout()).not.toContain('Commands:'); - expect(testIo.stdout()).not.toContain('setup demo'); - expect(testIo.stdout()).not.toContain('setup context'); - expect(testIo.stdout()).not.toContain('--skip-llm'); - expect(testIo.stdout()).not.toContain('--skip-embeddings'); - expect(testIo.stdout()).not.toContain('--embedding-model'); - expect(testIo.stdout()).not.toContain('--embedding-dimensions'); - expect(testIo.stdout()).not.toContain('--embedding-base-url'); + const stdout = testIo.stdout(); + expect(stdout).toContain('Usage: ktx setup [options]'); + expect(stdout).toContain('--agents'); + expect(stdout).toContain('--target '); + expect(stdout).toContain('--global'); + expect(stdout).toContain('--yes'); + expect(stdout).toContain('--no-input'); + expect(stdout).toContain('Global Options:'); + expect(stdout.match(/--project-dir /g)).toHaveLength(1); + expect(stdout).not.toContain('Commands:'); + expect(stdout).not.toContain('setup demo'); + expect(stdout).not.toContain('setup context'); + + for (const hiddenFlag of [ + '--new', + '--existing', + '--agent-scope', + '--skip-agents', + '--llm-backend', + '--anthropic-api-key-env', + '--vertex-project', + '--embedding-backend', + '--database ', + '--database-connection-id', + '--new-database-connection-id', + '--enable-historic-sql', + '--historic-sql-min-executions', + '--skip-databases', + '--source ', + '--source-connection-id', + '--metabase-database-id', + '--notion-root-page-id', + '--skip-initial-source-ingest', + '--skip-sources', + '--skip-llm', + '--skip-embeddings', + '--embedding-model', + '--embedding-dimensions', + '--embedding-base-url', + ]) { + expect(stdout).not.toContain(hiddenFlag); + } + expect(stdout).not.toMatch(/^ --project\s/m); expect(testIo.stderr()).toBe(''); }); @@ -725,6 +759,23 @@ describe('runKtxCli', () => { expect(setup).not.toHaveBeenCalled(); }); + it('rejects removed setup options', async () => { + const setup = vi.fn(async () => 0); + const cases = [ + ['setup', '--project'], + ['setup', '--agent-scope', 'global'], + ['setup', '--skip-initial-source-ingest'], + ]; + + for (const args of cases) { + const testIo = makeIo(); + await expect(runKtxCli(['--project-dir', tempDir, ...args], testIo.io, { setup })).resolves.toBe(1); + expect(testIo.stderr()).toMatch(/unknown option|error:/i); + } + + expect(setup).not.toHaveBeenCalled(); + }); + it('prints ingest help without invoking ingest execution', async () => { const testIo = makeIo(); const ingest = vi.fn(); @@ -961,7 +1012,7 @@ describe('runKtxCli', () => { expect(setup).not.toHaveBeenCalled(); expect(doctor).toHaveBeenCalledWith( - { command: 'project', projectDir: tempDir, outputMode: 'json', inputMode: 'disabled' }, + { command: 'project', projectDir: tempDir, outputMode: 'json', inputMode: 'disabled', verbose: false }, statusIo.io, ); expect(statusIo.stderr()).toBe(''); @@ -984,7 +1035,7 @@ describe('runKtxCli', () => { await expect(runKtxCli(['status', '--json', '--no-input'], statusIo.io, { doctor })).resolves.toBe(0); expect(doctor).toHaveBeenCalledWith( - { command: 'setup', outputMode: 'json', inputMode: 'disabled' }, + { command: 'setup', outputMode: 'json', inputMode: 'disabled', verbose: false }, statusIo.io, ); expect(statusIo.stderr()).toBe(''); @@ -1250,7 +1301,6 @@ describe('runKtxCli', () => { '--agents', '--target', 'codex', - '--project', '--no-input', '--yes', ], diff --git a/packages/cli/src/notion-page-picker-tui.test.tsx b/packages/cli/src/notion-page-picker-tui.test.tsx deleted file mode 100644 index 2d4dffc3..00000000 --- a/packages/cli/src/notion-page-picker-tui.test.tsx +++ /dev/null @@ -1,384 +0,0 @@ -/* @jsxImportSource react */ -import { render as renderInkTest } from 'ink-testing-library'; -import { act, type ReactNode } from 'react'; -import { afterEach, describe, expect, it, vi } from 'vitest'; -import { buildInitialState, buildPickerTree, type NotionPickerPageInput } from './notion-page-picker-tree.js'; -import { - NotionPickerApp, - notionPickerCommandForInkInput, - renderNotionPickerTui, - resolveNotionPickerWidth, - sanitizeNotionPickerTuiError, - windowItems, - windowOffset, - type NotionPickerInkInstance, - type NotionPickerInkRenderOptions, -} from './notion-page-picker-tui.js'; - -const IDS = { - engineering: '11111111-1111-1111-1111-111111111111', - architecture: '22222222-2222-2222-2222-222222222222', - marketing: '33333333-3333-3333-3333-333333333333', - finance: '44444444-4444-4444-4444-444444444444', - ops: '55555555-5555-5555-5555-555555555555', - sales: '66666666-6666-6666-6666-666666666666', - support: '77777777-7777-7777-7777-777777777777', - product: '88888888-8888-8888-8888-888888888888', - design: '99999999-9999-9999-9999-999999999999', -}; - -function pages(): NotionPickerPageInput[] { - return [ - { id: IDS.engineering, title: 'Engineering Docs', archived: false, parentId: null }, - { id: IDS.architecture, title: 'Architecture', archived: false, parentId: IDS.engineering }, - { id: IDS.marketing, title: 'Marketing', archived: false, parentId: null }, - ]; -} - -function manyPages(): NotionPickerPageInput[] { - return [ - { id: IDS.engineering, title: 'Engineering Docs', archived: false, parentId: null }, - { id: IDS.architecture, title: 'Architecture', archived: false, parentId: IDS.engineering }, - { id: IDS.marketing, title: 'Marketing', archived: false, parentId: null }, - { id: IDS.finance, title: 'Finance', archived: false, parentId: null }, - { id: IDS.ops, title: 'Operations', archived: false, parentId: null }, - { id: IDS.sales, title: 'Sales', archived: false, parentId: null }, - { id: IDS.support, title: 'Support', archived: false, parentId: null }, - { id: IDS.product, title: 'Product', archived: false, parentId: null }, - { id: IDS.design, title: 'Design', archived: false, parentId: null }, - ]; -} - -function state(mode: 'all_accessible' | 'selected_roots' = 'selected_roots') { - return buildInitialState({ - tree: buildPickerTree(pages()), - existingRootPageIds: [], - currentCrawlMode: mode, - }); -} - -async function waitForInkInput(): Promise { - await new Promise((resolve) => setTimeout(resolve, 10)); -} - -function fakeInkInstance(): NotionPickerInkInstance { - return { - rerender: vi.fn(), - unmount: vi.fn(), - waitUntilExit: vi.fn(async () => undefined), - }; -} - -function normalizeFrameWrap(frame: string | undefined): string { - return frame?.replace(/\n/g, ' ') ?? ''; -} - -afterEach(() => { - vi.useRealTimers(); -}); - -describe('notionPickerCommandForInkInput', () => { - it('maps browse, search, and confirm input to reducer commands', () => { - expect(notionPickerCommandForInkInput('', { downArrow: true }, state().search, null)).toBe('cursor-down'); - expect(notionPickerCommandForInkInput('', { upArrow: true }, state().search, null)).toBe('cursor-up'); - expect(notionPickerCommandForInkInput('', { rightArrow: true }, state().search, null)).toBe('cursor-right'); - expect(notionPickerCommandForInkInput('', { leftArrow: true }, state().search, null)).toBe('cursor-left'); - expect(notionPickerCommandForInkInput(' ', {}, state().search, null)).toBe('toggle-check'); - expect(notionPickerCommandForInkInput('/', {}, state().search, null)).toBe('search-start'); - expect(notionPickerCommandForInkInput('a', {}, state().search, null)).toBe('select-all-visible'); - expect(notionPickerCommandForInkInput('n', {}, state().search, null)).toBe('select-none'); - expect(notionPickerCommandForInkInput('s', {}, state().search, null)).toBe('save-request'); - expect(notionPickerCommandForInkInput('q', {}, state().search, null)).toBe('quit'); - expect(notionPickerCommandForInkInput('c', { ctrl: true }, state().search, null)).toBe('quit'); - - expect(notionPickerCommandForInkInput('x', {}, { editing: true, query: '' }, null)).toEqual({ - type: 'search-input', - value: 'x', - }); - expect(notionPickerCommandForInkInput('', { backspace: true }, { editing: true, query: 'x' }, null)).toBe( - 'search-backspace', - ); - expect(notionPickerCommandForInkInput('', { return: true }, { editing: true, query: 'x' }, null)).toBe( - 'search-submit', - ); - expect(notionPickerCommandForInkInput('', { escape: true }, { editing: true, query: 'x' }, null)).toBe( - 'search-cancel', - ); - - expect(notionPickerCommandForInkInput('y', {}, state().search, 'mode-switch')).toBe('save-confirm'); - expect(notionPickerCommandForInkInput('', { return: true }, state().search, 'mode-switch')).toBe('save-confirm'); - expect(notionPickerCommandForInkInput('n', {}, state().search, 'mode-switch')).toBe('save-cancel'); - }); -}); - -describe('window helpers', () => { - it('centers the selected row and returns the visible slice', () => { - expect(windowOffset(20, 10, 5)).toBe(8); - expect(windowItems(['a', 'b', 'c', 'd', 'e'], 3, 3)).toEqual({ items: ['c', 'd', 'e'], offset: 2 }); - }); - - it('clamps picker width to the design rule', () => { - expect(resolveNotionPickerWidth(200)).toBe(120); - expect(resolveNotionPickerWidth(100)).toBe(96); - expect(resolveNotionPickerWidth(50)).toBe(60); - expect(resolveNotionPickerWidth(undefined)).toBe(96); - }); -}); - -describe('NotionPickerApp', () => { - it('renders spec banners, row glyphs, search visibility, and hint text', () => { - const initialState = { - ...state('all_accessible'), - preLoadWarnings: ['1 stored root_page_ids no longer visible'], - }; - const { lastFrame } = renderInkTest( - , - ); - - const frame = lastFrame() ?? ''; - expect(frame).toContain('Notion pages visible to integration "Design Workspace"'); - expect(frame).toContain('5000-page cap reached - some pages not shown'); - expect(frame).toContain('1 stored root_page_ids no longer visible - they will be removed if you save'); - expect(frame).toContain('▸ [ ] Engineering Docs ▸ (1)'); - expect(frame).toContain(' [ ] Marketing'); - expect(frame).not.toContain('Search ready: -'); - expect(frame).toContain('space toggle · enter expand · / search · a all · n none · s save & exit · q quit'); - }); - - it('renders partial discovery warnings without stale-root save suffix', () => { - const initialState = { - ...state(), - preLoadWarnings: ['Notion search stopped early: rate limit after first page'], - }; - const { lastFrame } = renderInkTest( - , - ); - - const frame = lastFrame() ?? ''; - expect(frame).toContain('Notion search stopped early: rate limit after first page'); - expect(frame).not.toContain( - 'Notion search stopped early: rate limit after first page - they will be removed if you save', - ); - }); - - it('renders checked parents and locked descendants with the locked design glyphs', () => { - const initialState = { - ...state(), - checked: new Set([IDS.engineering]), - expanded: new Set([IDS.engineering]), - }; - const { lastFrame } = renderInkTest( - , - ); - - const frame = lastFrame() ?? ''; - expect(frame).toContain('▸ [×] Engineering Docs ▾'); - expect(frame).toContain(' [~] Architecture'); - }); - - it('supports keyboard selection, all_accessible confirmation, and save callback', async () => { - const onExit = vi.fn(); - const { stdin, lastFrame } = renderInkTest( - , - ); - - stdin.write(' '); - await waitForInkInput(); - expect(lastFrame()).toContain('[×] Engineering Docs'); - - stdin.write('s'); - await waitForInkInput(); - expect(normalizeFrameWrap(lastFrame())).toContain( - 'Save will switch crawl_mode all_accessible -> selected_roots and limit ingest to 1 selected page. [y] confirm [esc] back', - ); - - stdin.write('y'); - await waitForInkInput(); - expect(onExit).toHaveBeenCalledWith({ kind: 'save', rootPageIds: [IDS.engineering] }); - }); - - it('removes transient hints after their expiry time', async () => { - vi.useFakeTimers(); - const onExit = vi.fn(); - const { stdin, lastFrame } = renderInkTest( - , - ); - - await act(async () => { - stdin.write('s'); - await vi.advanceTimersByTimeAsync(10); - }); - expect(lastFrame()).toContain('Select at least one page or press q to quit'); - - await act(async () => { - await vi.advanceTimersByTimeAsync(2500); - }); - expect(lastFrame()).not.toContain('Select at least one page or press q to quit'); - expect(onExit).not.toHaveBeenCalled(); - }); - - it('renders row-window overflow indicators when the visible list is clipped', async () => { - const onExit = vi.fn(); - const initialState = buildInitialState({ - tree: buildPickerTree(manyPages()), - existingRootPageIds: [], - currentCrawlMode: 'selected_roots', - }); - initialState.expanded = new Set([IDS.engineering]); - const { stdin, lastFrame } = renderInkTest( - , - ); - - expect(lastFrame()).toContain('↓ 4 more'); - - stdin.write('\u001B[B'); - stdin.write('\u001B[B'); - stdin.write('\u001B[B'); - stdin.write('\u001B[B'); - await waitForInkInput(); - - const frame = lastFrame() ?? ''; - expect(frame).toContain('↑ '); - expect(frame).toContain('↓ '); - expect(onExit).not.toHaveBeenCalled(); - }); - - it('returns quit without saving', async () => { - const onExit = vi.fn(); - const { stdin } = renderInkTest( - , - ); - - stdin.write('q'); - await waitForInkInput(); - expect(onExit).toHaveBeenCalledWith({ kind: 'quit' }); - }); -}); - -describe('renderNotionPickerTui', () => { - it('returns the app result from the Ink runtime', async () => { - const io = { - stdin: { isTTY: true, setRawMode: vi.fn() }, - stdout: { isTTY: true, columns: 100, rows: 24, write: vi.fn() }, - stderr: { write: vi.fn() }, - }; - const renderInk = vi.fn((_tree: ReactNode, _options: NotionPickerInkRenderOptions) => fakeInkInstance()); - - await expect( - renderNotionPickerTui( - { - initialState: state(), - connectionId: 'notion-main', - workspaceLabel: 'Design Workspace', - cappedAtCount: null, - currentCrawlMode: 'selected_roots', - }, - io, - { renderInk }, - ), - ).resolves.toEqual({ kind: 'quit' }); - expect(renderInk).toHaveBeenCalledOnce(); - }); - - it('sanitizes render errors and tells the user to use no-input mode', async () => { - expect(sanitizeNotionPickerTuiError(new Error('token=secret https://api.notion.com/v1/search'))).toBe( - '[redacted] [redacted-url]', - ); - }); - - it('falls back to quit with a scripted-mode hint when Ink cannot initialize', async () => { - let stderr = ''; - const io = { - stdin: { isTTY: false, setRawMode: vi.fn() }, - stdout: { isTTY: false, columns: 100, rows: 24, write: vi.fn() }, - stderr: { - write(chunk: string) { - stderr += chunk; - }, - }, - }; - - await expect( - renderNotionPickerTui( - { - initialState: state(), - connectionId: 'notion-main', - workspaceLabel: 'Design Workspace', - cappedAtCount: null, - currentCrawlMode: 'selected_roots', - }, - io, - { - renderInk: vi.fn(() => { - throw new Error('token=secret'); - }), - }, - ), - ).resolves.toEqual({ kind: 'quit' }); - expect(stderr).toContain('Use --no-input --notion-root-page-id for scripted mode'); - expect(stderr).not.toContain('secret'); - }); -}); diff --git a/packages/cli/src/notion-page-picker.test.ts b/packages/cli/src/notion-page-picker.test.ts index 77710716..29f5a352 100644 --- a/packages/cli/src/notion-page-picker.test.ts +++ b/packages/cli/src/notion-page-picker.test.ts @@ -1,4 +1,6 @@ import { describe, expect, it, vi } from 'vitest'; +import type { PickerState } from './tree-picker-state.js'; +import type { TreePickerChrome, TreePickerResult, TreePickerTuiIo } from './tree-picker-tui.js'; import { discoverNotionPickerPages, notionPickerPageFromSearchResult, @@ -6,8 +8,6 @@ import { pickNotionRootPages, resolveNotionWorkspaceLabel, type NotionPickerApi, - type PickerRenderInput, - type PickerRenderResult, } from './notion-page-picker.js'; function makeIo() { @@ -162,20 +162,27 @@ describe('Notion page picker helpers', () => { }); }); +type RenderPickerArgs = [TreePickerChrome, PickerState, TreePickerTuiIo]; + describe('pickNotionRootPages', () => { it('discovers visible pages, warns about stale roots, renders the TUI, and returns selected roots', async () => { const api = fakeNotionApi([ notionPage(PAGE_IDS.engineering, 'Engineering'), notionPage(PAGE_IDS.architecture, 'Architecture', PAGE_IDS.engineering), ]); - const renderPicker = vi.fn(async (input: PickerRenderInput): Promise => { - expect(input.connectionId).toBe('notion-main'); - expect(input.workspaceLabel).toBe('Design Workspace'); - expect(input.currentCrawlMode).toBe('all_accessible'); - expect(input.cappedAtCount).toBeNull(); - expect(input.initialState.preLoadWarnings).toEqual(['1 stored root_page_ids no longer visible']); - return { kind: 'save', rootPageIds: [PAGE_IDS.engineering] }; - }); + const renderPicker = vi.fn( + async (chrome: TreePickerChrome, state: PickerState): Promise => { + expect(chrome.title).toBe('Select Notion pages to ingest'); + expect(chrome.subtitleLines).toEqual(['Workspace: Design Workspace']); + expect(chrome.warningLines ?? []).toEqual([]); + expect(chrome.confirmSaveMessage).toBeTypeOf('function'); + expect(state.requireConfirmOnSave).toBe(true); + expect(state.preLoadWarnings).toEqual([ + '1 stored root_page_ids no longer visible - they will be removed if you save', + ]); + return { kind: 'save', selectedIds: [PAGE_IDS.engineering] }; + }, + ); const io = makeIo(); await expect( @@ -223,7 +230,7 @@ describe('pickNotionRootPages', () => { makeIo().io, { createNotionApi, - renderPicker: vi.fn(async (): Promise => ({ kind: 'quit' })), + renderPicker: vi.fn(async (): Promise => ({ kind: 'quit' })), }, ), ).resolves.toEqual({ kind: 'back' }); @@ -243,11 +250,13 @@ describe('pickNotionRootPages', () => { .mockRejectedValueOnce(new Error('rate limit after first page')), retrieveBotUser: vi.fn(async () => ({ name: 'Notion bot', bot: { workspace_name: 'Design Workspace' } })), }; - let renderInput: PickerRenderInput | undefined; - const renderPicker = vi.fn(async (input: PickerRenderInput): Promise => { - renderInput = input; - return { kind: 'quit' }; - }); + let captured: RenderPickerArgs | undefined; + const renderPicker = vi.fn( + async (chrome: TreePickerChrome, state: PickerState, io: TreePickerTuiIo): Promise => { + captured = [chrome, state, io]; + return { kind: 'quit' }; + }, + ); const io = makeIo(); await expect( @@ -271,11 +280,12 @@ describe('pickNotionRootPages', () => { ).resolves.toEqual({ kind: 'back' }); expect(renderPicker).toHaveBeenCalledOnce(); - if (!renderInput) { + if (!captured) { throw new Error('renderPicker was not called'); } - expect(renderInput.initialState.preLoadWarnings).toEqual(['Notion search stopped early: rate limit after first page']); - expect(renderInput.initialState.tree.map((node) => node.title)).toEqual(['Engineering']); + const [, state] = captured; + expect(state.preLoadWarnings).toEqual(['Notion search stopped early: rate limit after first page']); + expect(state.tree.map((node) => node.title)).toEqual(['Engineering']); expect(io.stderr()).toContain('Notion search stopped early: rate limit after first page'); }); @@ -300,7 +310,7 @@ describe('pickNotionRootPages', () => { }), retrieveBotUser: vi.fn(async () => ({ name: 'Notion bot' })), })), - renderPicker: vi.fn(async (): Promise => ({ kind: 'quit' })), + renderPicker: vi.fn(async (): Promise => ({ kind: 'quit' })), }, ), ).resolves.toEqual({ kind: 'unavailable', message: 'Notion API unavailable' }); diff --git a/packages/cli/src/notion-page-picker.ts b/packages/cli/src/notion-page-picker.ts index 807c0fc0..26e561f5 100644 --- a/packages/cli/src/notion-page-picker.ts +++ b/packages/cli/src/notion-page-picker.ts @@ -3,13 +3,19 @@ import { type NotionApi, type NotionBotInfo, NotionClient } from '@ktx/context/i import type { KtxProjectConnectionConfig } from '@ktx/context/project'; import type { KtxCliIo } from './cli-runtime.js'; import { profileMark } from './startup-profile.js'; -import { buildInitialState, buildPickerTree, type NotionPickerPageInput } from './notion-page-picker-tree.js'; import { - type NotionPickerTuiIo, - type PickerRenderInput, - type PickerRenderResult, - renderNotionPickerTui, -} from './notion-page-picker-tui.js'; + buildInitialState, + buildPickerTree, + flattenSelection, + type PickerState, + type TreePickerNodeInput, +} from './tree-picker-state.js'; +import { + renderTreePickerTui, + type TreePickerChrome, + type TreePickerResult, + type TreePickerTuiIo, +} from './tree-picker-tui.js'; profileMark('module:notion-page-picker'); @@ -19,8 +25,6 @@ export interface PickNotionRootPagesArgs { } export type NotionPickerApi = Pick; -export type { PickerRenderInput, PickerRenderResult }; - export type NotionRootPagePickResult = | { kind: 'selected'; rootPageIds: string[] } | { kind: 'back' } @@ -29,10 +33,16 @@ export type NotionRootPagePickResult = export interface NotionRootPagePickerDeps { env?: Record; createNotionApi?: (authToken: string) => NotionPickerApi; - renderPicker?: (input: PickerRenderInput, io: NotionPickerTuiIo) => Promise; + renderPicker?: ( + chrome: TreePickerChrome, + initialState: PickerState, + io: TreePickerTuiIo, + ) => Promise; } const NOTION_PICKER_PAGE_CAP = 5000; +const NOTION_SCRIPTED_MODE_HINT = + 'Notion picker requires a TTY. Use --no-input --notion-root-page-id for scripted mode.'; function assertSafeNotionPickerConnectionId(connectionId: string): void { if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(connectionId)) { @@ -50,6 +60,14 @@ export function normalizeNotionPageId(value: string): string { return `${lower.slice(0, 8)}-${lower.slice(8, 12)}-${lower.slice(12, 16)}-${lower.slice(16, 20)}-${lower.slice(20)}`; } +function tryNormalizeNotionPageId(value: string): string | null { + try { + return normalizeNotionPageId(value); + } catch { + return null; + } +} + function recordValue(value: unknown): Record | null { return typeof value === 'object' && value !== null && !Array.isArray(value) ? (value as Record) @@ -88,7 +106,7 @@ function extractParentPageId(page: Record): string | null { return normalizeNotionPageId(parent.page_id); } -export function notionPickerPageFromSearchResult(result: Record): NotionPickerPageInput { +export function notionPickerPageFromSearchResult(result: Record): TreePickerNodeInput { const id = typeof result.id === 'string' ? normalizeNotionPageId(result.id) : ''; if (!id) { throw new Error('Notion page search result is missing id'); @@ -104,9 +122,9 @@ export function notionPickerPageFromSearchResult(result: Record export async function discoverNotionPickerPages( api: NotionPickerApi, options: { cap?: number } = {}, -): Promise<{ pages: NotionPickerPageInput[]; cappedAtCount: number | null; warnings: string[] }> { +): Promise<{ pages: TreePickerNodeInput[]; cappedAtCount: number | null; warnings: string[] }> { const cap = options.cap ?? NOTION_PICKER_PAGE_CAP; - const pages: NotionPickerPageInput[] = []; + const pages: TreePickerNodeInput[] = []; const warnings: string[] = []; let cursor: string | null | undefined = null; @@ -171,6 +189,33 @@ function notionCrawlMode(connection: KtxProjectConnectionConfig): 'all_accessibl return connection.crawl_mode === 'all_accessible' ? 'all_accessible' : 'selected_roots'; } +function selectedPageCountText(count: number): string { + return `${count} selected ${count === 1 ? 'page' : 'pages'}`; +} + +function notionChrome(args: { + workspaceLabel: string; + cappedAtCount: number | null; + currentCrawlMode: 'all_accessible' | 'selected_roots'; +}): TreePickerChrome { + const warningLines: string[] = []; + if (args.cappedAtCount) { + warningLines.push(`${args.cappedAtCount}-page cap reached - some pages not shown`); + } + return { + title: 'Select Notion pages to ingest', + subtitleLines: [`Workspace: ${args.workspaceLabel}`], + warningLines, + confirmSaveMessage: + args.currentCrawlMode === 'all_accessible' + ? (state) => + `Switch crawl_mode from all_accessible to selected_roots? Will limit ingest to ${selectedPageCountText( + flattenSelection(state.checked, state.byId).length, + )}. Press Enter to confirm or Escape to go back.` + : undefined, + }; +} + export async function pickNotionRootPages( args: PickNotionRootPagesArgs, io: KtxCliIo = process, @@ -190,10 +235,14 @@ export async function pickNotionRootPages( const api = deps.createNotionApi ? deps.createNotionApi(authToken) : new NotionClient(authToken); const discovery = await discoverNotionPickerPages(api); const tree = buildPickerTree(discovery.pages); + const normalizedExistingIds = stringArray(args.connection.root_page_ids) + .map((raw) => tryNormalizeNotionPageId(raw)) + .filter((id): id is string => id !== null); const initialState = buildInitialState({ tree, - existingRootPageIds: stringArray(args.connection.root_page_ids), - currentCrawlMode: crawlMode, + existingSelectedIds: normalizedExistingIds, + requireConfirmOnSave: crawlMode === 'all_accessible', + staleWarning: (count) => `${count} stored root_page_ids no longer visible - they will be removed if you save`, }); const preLoadWarnings = [...discovery.warnings, ...initialState.preLoadWarnings]; const renderState = @@ -207,23 +256,25 @@ export async function pickNotionRootPages( io.stderr.write(`${warning}\n`); } const workspaceLabel = await resolveNotionWorkspaceLabel(api, args.connectionId); - const result = await (deps.renderPicker ?? renderNotionPickerTui)( - { - initialState: renderState, - connectionId: args.connectionId, - workspaceLabel, - cappedAtCount: discovery.cappedAtCount, - currentCrawlMode: crawlMode, - }, - io as NotionPickerTuiIo, - ); + const chrome = notionChrome({ + workspaceLabel, + cappedAtCount: discovery.cappedAtCount, + currentCrawlMode: crawlMode, + }); + const renderPicker = + deps.renderPicker ?? + ((chromeArg, state, ioArg) => + renderTreePickerTui({ chrome: chromeArg, initialState: state }, ioArg, { + scriptedModeHint: NOTION_SCRIPTED_MODE_HINT, + })); + const result = await renderPicker(chrome, renderState, io as TreePickerTuiIo); if (result.kind === 'quit') { return { kind: 'back' }; } - if (result.rootPageIds.length === 0) { + if (result.selectedIds.length === 0) { return { kind: 'unavailable', message: 'Notion picker did not return any selected pages.' }; } - return { kind: 'selected', rootPageIds: result.rootPageIds }; + return { kind: 'selected', rootPageIds: result.selectedIds }; } catch (error) { return { kind: 'unavailable', message: error instanceof Error ? error.message : String(error) }; } diff --git a/packages/cli/src/setup-agents.ts b/packages/cli/src/setup-agents.ts index 9505307d..7a18a969 100644 --- a/packages/cli/src/setup-agents.ts +++ b/packages/cli/src/setup-agents.ts @@ -82,7 +82,7 @@ export function plannedKtxAgentFiles(input: { { kind: 'file', path: join(codexHome, 'instructions/ktx.md'), role: 'rule' as const }, ]; } - throw new Error(`Global ${input.target} installation is not supported; use --project.`); + throw new Error(`Global ${input.target} installation is not supported; omit --global.`); } const root = resolve(input.projectDir); diff --git a/packages/cli/src/setup-databases.test.ts b/packages/cli/src/setup-databases.test.ts index d010a908..d3a55fba 100644 --- a/packages/cli/src/setup-databases.test.ts +++ b/packages/cli/src/setup-databases.test.ts @@ -5,10 +5,15 @@ import { initKtxProject, parseKtxProjectConfig, readKtxSetupState, writeKtxSetup import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { type KtxSetupDatabaseDriver, + type KtxSetupDatabasesDeps, type KtxSetupDatabasesPromptAdapter, runKtxSetupDatabasesStep, } from './setup-databases.js'; import type { KtxCliIo } from './cli-runtime.js'; +import type { + DatabaseScopePickResult, + PickDatabaseScopeArgs, +} from './database-tree-picker.js'; function makeIo() { let stdout = ''; @@ -32,6 +37,43 @@ function makeIo() { }; } +type ScopePick = + | 'back' + | 'enable-all' + | { schemas: string[]; tables: string[] }; + +interface PickerStubs { + pickDatabaseScope: KtxSetupDatabasesDeps['pickDatabaseScope']; + scopeCalls: PickDatabaseScopeArgs[]; +} + +function makePickerStubs(options: { scopes?: ScopePick[] } = {}): PickerStubs { + const queue: ScopePick[] = [...(options.scopes ?? [])]; + const scopeCalls: PickDatabaseScopeArgs[] = []; + return { + scopeCalls, + pickDatabaseScope: vi.fn(async (args: PickDatabaseScopeArgs): Promise => { + scopeCalls.push(args); + const next = queue.shift(); + if (next === undefined || next === 'enable-all') { + const enabledTables = args.discovered.map((t) => `${t.schema}.${t.name}`); + const activeSchemas = args.supportsSchemaScope + ? Array.from(new Set(args.discovered.map((t) => t.schema))) + : []; + return { kind: 'selected', activeSchemas, enabledTables }; + } + if (next === 'back') { + return { kind: 'back' }; + } + return { + kind: 'selected', + activeSchemas: args.supportsSchemaScope ? next.schemas : [], + enabledTables: next.tables, + }; + }), + }; +} + function makePromptAdapter(options: { multiselectValues?: string[][]; selectValues?: string[]; @@ -240,8 +282,9 @@ describe('setup databases step', () => { expect(prompts.select).toHaveBeenCalledWith({ message: 'Configure PostgreSQL', options: [ - { value: 'existing:warehouse', label: 'Use existing PostgreSQL connection: warehouse' }, - { value: 'new', label: 'Add new PostgreSQL connection' }, + { value: 'existing:warehouse', label: 'Keep existing PostgreSQL connection: warehouse' }, + { value: 'edit:warehouse', label: 'Edit PostgreSQL connection: warehouse' }, + { value: 'new', label: 'Add another PostgreSQL connection' }, { value: 'back', label: 'Back' }, ], }); @@ -564,7 +607,8 @@ describe('setup databases step', () => { message: 'Primary sources already configured: warehouse\nWhat would you like to do?', options: [ { value: 'continue', label: 'Continue to knowledge sources' }, - { value: 'add', label: 'Add another primary source' }, + { value: 'edit', label: 'Edit an existing primary source' }, + { value: 'add', label: 'Add additional primary sources' }, ], }); expect(testConnection).not.toHaveBeenCalled(); @@ -608,11 +652,16 @@ describe('setup databases step', () => { connectionIds: ['warehouse', 'mysql-warehouse'], }); expect(prompts.multiselect).toHaveBeenCalledTimes(1); + expect(prompts.multiselect).toHaveBeenCalledWith(expect.objectContaining({ + initialValues: ['postgres'], + required: true, + })); expect(prompts.select).toHaveBeenCalledWith({ message: 'Primary sources already configured: warehouse\nWhat would you like to do?', options: [ { value: 'continue', label: 'Continue to knowledge sources' }, - { value: 'add', label: 'Add another primary source' }, + { value: 'edit', label: 'Edit an existing primary source' }, + { value: 'add', label: 'Add additional primary sources' }, ], }); expect(testConnection).toHaveBeenCalledTimes(1); @@ -642,11 +691,16 @@ describe('setup databases step', () => { connectionIds: ['postgres-warehouse', 'mysql-warehouse'], }); expect(prompts.multiselect).toHaveBeenCalledTimes(2); + expect(prompts.multiselect).toHaveBeenNthCalledWith(2, expect.objectContaining({ + initialValues: ['postgres'], + required: true, + })); expect(prompts.select).toHaveBeenCalledWith({ message: 'Primary sources already configured: postgres-warehouse\nWhat would you like to do?', options: [ { value: 'continue', label: 'Continue to knowledge sources' }, - { value: 'add', label: 'Add another primary source' }, + { value: 'edit', label: 'Edit an existing primary source' }, + { value: 'add', label: 'Add additional primary sources' }, ], }); const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); @@ -675,12 +729,17 @@ describe('setup databases step', () => { connectionIds: ['postgres-warehouse'], }); expect(prompts.multiselect).toHaveBeenCalledTimes(2); + expect(prompts.multiselect).toHaveBeenNthCalledWith(2, expect.objectContaining({ + initialValues: ['postgres'], + required: true, + })); expect(io.stdout()).not.toContain('KTX cannot work without at least one primary source'); expect(prompts.select).toHaveBeenNthCalledWith(2, { message: 'Primary sources already configured: postgres-warehouse\nWhat would you like to do?', options: [ { value: 'continue', label: 'Continue to knowledge sources' }, - { value: 'add', label: 'Add another primary source' }, + { value: 'edit', label: 'Edit an existing primary source' }, + { value: 'add', label: 'Add additional primary sources' }, ], }); }); @@ -715,16 +774,403 @@ describe('setup databases step', () => { ); expect(result).toEqual({ status: 'ready', projectDir: tempDir, connectionIds: ['warehouse'] }); + expect(prompts.multiselect).toHaveBeenCalledWith(expect.objectContaining({ + initialValues: ['postgres'], + required: true, + })); expect(io.stdout()).not.toContain('KTX cannot work without at least one primary source'); expect(prompts.select).toHaveBeenNthCalledWith(2, { message: 'Primary sources already configured: warehouse\nWhat would you like to do?', options: [ { value: 'continue', label: 'Continue to knowledge sources' }, - { value: 'add', label: 'Add another primary source' }, + { value: 'edit', label: 'Edit an existing primary source' }, + { value: 'add', label: 'Add additional primary sources' }, ], }); }); + it('returns from primary source edit selection back to the configured source menu', async () => { + await writeFile( + join(tempDir, 'ktx.yaml'), + [ + 'project: warehouse', + 'connections:', + ' warehouse:', + ' driver: postgres', + ' url: env:DATABASE_URL', + 'setup:', + ' database_connection_ids:', + ' - warehouse', + '', + ].join('\n'), + 'utf-8', + ); + await writeKtxSetupState(tempDir, { completed_steps: ['databases'] }); + const prompts = makePromptAdapter({ + selectValues: ['edit', 'back', 'continue'], + }); + const testConnection = vi.fn(async () => 0); + const scanConnection = vi.fn(async () => 0); + + const result = await runKtxSetupDatabasesStep( + { projectDir: tempDir, inputMode: 'auto', skipDatabases: false, databaseSchemas: [] }, + makeIo().io, + { prompts, testConnection, scanConnection }, + ); + + expect(result).toEqual({ status: 'ready', projectDir: tempDir, connectionIds: ['warehouse'] }); + expect(prompts.select).toHaveBeenNthCalledWith(2, { + message: 'Primary source to edit', + options: [ + { value: 'warehouse', label: 'warehouse (PostgreSQL)' }, + { value: 'back', label: 'Back' }, + ], + }); + expect(prompts.select).toHaveBeenNthCalledWith(3, { + message: 'Primary sources already configured: warehouse\nWhat would you like to do?', + options: [ + { value: 'continue', label: 'Continue to knowledge sources' }, + { value: 'edit', label: 'Edit an existing primary source' }, + { value: 'add', label: 'Add additional primary sources' }, + ], + }); + expect(testConnection).not.toHaveBeenCalled(); + expect(scanConnection).not.toHaveBeenCalled(); + }); + + it('reruns table selection after editing schema scope so stale enabled tables are removed', async () => { + await writeFile( + join(tempDir, 'ktx.yaml'), + [ + 'project: warehouse', + 'connections:', + ' warehouse:', + ' driver: postgres', + ' url: env:DATABASE_URL', + ' schemas:', + ' - public', + ' enabled_tables:', + ' - public.orders', + 'setup:', + ' database_connection_ids:', + ' - warehouse', + '', + ].join('\n'), + 'utf-8', + ); + await writeKtxSetupState(tempDir, { completed_steps: ['databases'] }); + const prompts = makePromptAdapter({ + textValues: ['env:DATABASE_URL'], + }); + let primaryMenuCount = 0; + vi.mocked(prompts.select).mockImplementation(async (options) => { + if (options.message === 'Primary sources already configured: warehouse\nWhat would you like to do?') { + primaryMenuCount += 1; + return primaryMenuCount === 1 ? 'edit' : 'continue'; + } + if (options.message === 'Primary source to edit') return 'warehouse'; + if (options.message === 'How do you want to connect to PostgreSQL?') return 'url'; + return 'back'; + }); + const testConnection = vi.fn(async () => 0); + const scanConnection = vi.fn(async () => 0); + const listSchemas = vi.fn(async () => ['analytics', 'public']); + const listTables = vi.fn(async () => [{ schema: 'analytics', name: 'customers', kind: 'table' as const }]); + const pickers = makePickerStubs({ + scopes: [{ schemas: ['analytics'], tables: ['analytics.customers'] }], + }); + + const result = await runKtxSetupDatabasesStep( + { projectDir: tempDir, inputMode: 'auto', skipDatabases: false, databaseSchemas: [] }, + makeIo().io, + { + prompts, + testConnection, + scanConnection, + listSchemas, + listTables, + pickDatabaseScope: pickers.pickDatabaseScope, + }, + ); + + expect(result).toEqual({ status: 'ready', projectDir: tempDir, connectionIds: ['warehouse'] }); + expect(prompts.text).toHaveBeenCalledWith({ + message: textInputPrompt('PostgreSQL connection URL'), + placeholder: 'env:DATABASE_URL', + initialValue: 'env:DATABASE_URL', + }); + expect(listTables).toHaveBeenCalledWith(tempDir, 'warehouse', ['analytics', 'public']); + expect(testConnection).toHaveBeenCalledWith(tempDir, 'warehouse', expect.anything()); + expect(scanConnection).toHaveBeenCalledWith(tempDir, 'warehouse', expect.anything()); + const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + expect(config.connections.warehouse).toMatchObject({ + schemas: ['analytics'], + enabled_tables: ['analytics.customers'], + }); + }); + + it('preselects existing schema and table choices when editing a primary source', async () => { + await writeFile( + join(tempDir, 'ktx.yaml'), + [ + 'project: warehouse', + 'connections:', + ' warehouse:', + ' driver: postgres', + ' url: env:DATABASE_URL', + ' schemas:', + ' - public', + ' enabled_tables:', + ' - public.customers', + ' - public.orders', + 'setup:', + ' database_connection_ids:', + ' - warehouse', + '', + ].join('\n'), + 'utf-8', + ); + await writeKtxSetupState(tempDir, { completed_steps: ['databases'] }); + const prompts = makePromptAdapter({ + textValues: ['env:DATABASE_URL'], + }); + let primaryMenuCount = 0; + vi.mocked(prompts.select).mockImplementation(async (options) => { + if (options.message === 'Primary sources already configured: warehouse\nWhat would you like to do?') { + primaryMenuCount += 1; + return primaryMenuCount === 1 ? 'edit' : 'continue'; + } + if (options.message === 'Primary source to edit') return 'warehouse'; + if (options.message === 'How do you want to connect to PostgreSQL?') return 'url'; + return 'back'; + }); + const listSchemas = vi.fn(async () => ['orbit_analytics', 'orbit_raw', 'public']); + const listTables = vi.fn(async () => [ + { schema: 'public', name: 'customers', kind: 'table' as const }, + { schema: 'public', name: 'orders', kind: 'table' as const }, + { schema: 'public', name: 'products', kind: 'table' as const }, + ]); + const pickers = makePickerStubs({ + scopes: [{ schemas: ['public'], tables: ['public.customers', 'public.orders'] }], + }); + + const result = await runKtxSetupDatabasesStep( + { projectDir: tempDir, inputMode: 'auto', skipDatabases: false, databaseSchemas: [] }, + makeIo().io, + { + prompts, + testConnection: vi.fn(async () => 0), + scanConnection: vi.fn(async () => 0), + listSchemas, + listTables, + pickDatabaseScope: pickers.pickDatabaseScope, + }, + ); + + expect(result).toEqual({ status: 'ready', projectDir: tempDir, connectionIds: ['warehouse'] }); + expect(pickers.scopeCalls).toHaveLength(1); + expect(pickers.scopeCalls[0]).toMatchObject({ + connectionId: 'warehouse', + schemaNoun: 'schema', + supportsSchemaScope: true, + existing: { enabledTables: ['public.customers', 'public.orders'] }, + }); + const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + expect(config.connections.warehouse).toMatchObject({ + schemas: ['public'], + enabled_tables: ['public.customers', 'public.orders'], + }); + }); + + it('returns to the configured primary menu when backing out of schema review during edit', async () => { + await writeFile( + join(tempDir, 'ktx.yaml'), + [ + 'project: warehouse', + 'connections:', + ' warehouse:', + ' driver: postgres', + ' url: env:DATABASE_URL', + ' schemas:', + ' - public', + ' enabled_tables:', + ' - public.orders', + 'setup:', + ' database_connection_ids:', + ' - warehouse', + '', + ].join('\n'), + 'utf-8', + ); + await writeKtxSetupState(tempDir, { completed_steps: ['databases'] }); + const prompts = makePromptAdapter({ + textValues: ['env:DATABASE_URL'], + }); + let primaryMenuCount = 0; + vi.mocked(prompts.select).mockImplementation(async (options) => { + if (options.message === 'Primary sources already configured: warehouse\nWhat would you like to do?') { + primaryMenuCount += 1; + return primaryMenuCount === 1 ? 'edit' : 'continue'; + } + if (options.message === 'Primary source to edit') return 'warehouse'; + if (options.message === 'How do you want to connect to PostgreSQL?') return 'url'; + return 'back'; + }); + const testConnection = vi.fn(async () => 0); + const scanConnection = vi.fn(async () => 0); + const listSchemas = vi.fn(async () => ['analytics', 'public']); + const listTables = vi.fn(async () => [ + { schema: 'analytics', name: 'customers', kind: 'table' as const }, + { schema: 'public', name: 'orders', kind: 'table' as const }, + ]); + const pickers = makePickerStubs({ scopes: ['back'] }); + + const result = await runKtxSetupDatabasesStep( + { projectDir: tempDir, inputMode: 'auto', skipDatabases: false, databaseSchemas: [] }, + makeIo().io, + { + prompts, + testConnection, + scanConnection, + listSchemas, + listTables, + pickDatabaseScope: pickers.pickDatabaseScope, + }, + ); + + expect(result).toEqual({ status: 'ready', projectDir: tempDir, connectionIds: ['warehouse'] }); + expect(primaryMenuCount).toBe(2); + expect(testConnection).toHaveBeenCalledWith(tempDir, 'warehouse', expect.anything()); + expect(scanConnection).not.toHaveBeenCalled(); + const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + expect(config.connections.warehouse).toMatchObject({ + url: 'env:DATABASE_URL', + schemas: ['public'], + enabled_tables: ['public.orders'], + }); + }); + + it('returns to the configured primary menu when backing out of table review during edit', async () => { + await writeFile( + join(tempDir, 'ktx.yaml'), + [ + 'project: warehouse', + 'connections:', + ' warehouse:', + ' driver: postgres', + ' url: env:DATABASE_URL', + ' schemas:', + ' - public', + ' enabled_tables:', + ' - public.orders', + 'setup:', + ' database_connection_ids:', + ' - warehouse', + '', + ].join('\n'), + 'utf-8', + ); + await writeKtxSetupState(tempDir, { completed_steps: ['databases'] }); + const prompts = makePromptAdapter({ textValues: ['env:DATABASE_URL'] }); + let primaryMenuCount = 0; + vi.mocked(prompts.select).mockImplementation(async (options) => { + if (options.message === 'Primary sources already configured: warehouse\nWhat would you like to do?') { + primaryMenuCount += 1; + return primaryMenuCount === 1 ? 'edit' : 'continue'; + } + if (options.message === 'Primary source to edit') return 'warehouse'; + if (options.message === 'How do you want to connect to PostgreSQL?') return 'url'; + return 'back'; + }); + const testConnection = vi.fn(async () => 0); + const scanConnection = vi.fn(async () => 0); + const listSchemas = vi.fn(async () => ['public']); + const listTables = vi.fn(async () => [ + { schema: 'public', name: 'customers', kind: 'table' as const }, + { schema: 'public', name: 'orders', kind: 'table' as const }, + ]); + const pickers = makePickerStubs({ scopes: ['back'] }); + + const result = await runKtxSetupDatabasesStep( + { projectDir: tempDir, inputMode: 'auto', skipDatabases: false, databaseSchemas: [] }, + makeIo().io, + { + prompts, + testConnection, + scanConnection, + listSchemas, + listTables, + pickDatabaseScope: pickers.pickDatabaseScope, + }, + ); + + expect(result).toEqual({ status: 'ready', projectDir: tempDir, connectionIds: ['warehouse'] }); + expect(primaryMenuCount).toBe(2); + expect(listTables).toHaveBeenCalledWith(tempDir, 'warehouse', ['public']); + expect(scanConnection).not.toHaveBeenCalled(); + const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + expect(config.connections.warehouse).toMatchObject({ + url: 'env:DATABASE_URL', + schemas: ['public'], + enabled_tables: ['public.orders'], + }); + }); + + it('restores an existing primary source edit when the follow-up scan fails', async () => { + await writeFile( + join(tempDir, 'ktx.yaml'), + [ + 'project: warehouse', + 'connections:', + ' warehouse:', + ' driver: postgres', + ' url: env:DATABASE_URL', + ' schemas:', + ' - public', + ' enabled_tables:', + ' - public.orders', + 'setup:', + ' database_connection_ids:', + ' - warehouse', + '', + ].join('\n'), + 'utf-8', + ); + await writeKtxSetupState(tempDir, { completed_steps: ['databases'] }); + const prompts = makePromptAdapter({ + textValues: ['env:DATABASE_URL'], + }); + vi.mocked(prompts.select).mockImplementation(async (options) => { + if (options.message === 'Primary sources already configured: warehouse\nWhat would you like to do?') return 'edit'; + if (options.message === 'Primary source to edit') return 'warehouse'; + if (options.message === 'How do you want to connect to PostgreSQL?') return 'url'; + return 'back'; + }); + const listTables = vi.fn(async () => [ + { schema: 'public', name: 'customers', kind: 'table' as const }, + { schema: 'public', name: 'orders', kind: 'table' as const }, + ]); + const pickers = makePickerStubs({ scopes: ['enable-all'] }); + + const result = await runKtxSetupDatabasesStep( + { projectDir: tempDir, inputMode: 'auto', skipDatabases: false, databaseSchemas: [] }, + makeIo().io, + { + prompts, + testConnection: vi.fn(async () => 0), + scanConnection: vi.fn(async () => 1), + listTables, + pickDatabaseScope: pickers.pickDatabaseScope, + }, + ); + + expect(result).toEqual({ status: 'failed', projectDir: tempDir }); + const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + expect(config.connections.warehouse).toMatchObject({ + enabled_tables: ['public.orders'], + }); + }); + it('lets Escape from connection fields return to connection method selection', async () => { const prompts = makePromptAdapter({ selectValues: ['fields', 'url'], @@ -1000,7 +1446,6 @@ describe('setup databases step', () => { const prompts = makePromptAdapter({ selectValues: ['url'], textValues: ['', 'env:DATABASE_URL'], - multiselectValues: [['orbit_analytics', 'orbit_raw']], }); const testConnection = vi.fn(async () => 0); const scanConnection = vi.fn(async asyncScanProjectDir => { @@ -1011,6 +1456,19 @@ describe('setup databases step', () => { return 0; }); const listSchemas = vi.fn(async () => ['orbit_analytics', 'orbit_raw', 'public']); + const listTables = vi.fn(async () => [ + { schema: 'orbit_analytics', name: 'events', kind: 'table' as const }, + { schema: 'orbit_raw', name: 'inputs', kind: 'table' as const }, + { schema: 'public', name: 'misc', kind: 'table' as const }, + ]); + const pickers = makePickerStubs({ + scopes: [ + { + schemas: ['orbit_analytics', 'orbit_raw'], + tables: ['orbit_analytics.events', 'orbit_raw.inputs'], + }, + ], + }); const result = await runKtxSetupDatabasesStep( { @@ -1021,20 +1479,24 @@ describe('setup databases step', () => { skipDatabases: false, }, io.io, - { prompts, testConnection, scanConnection, listSchemas }, + { + prompts, + testConnection, + scanConnection, + listSchemas, + listTables, + pickDatabaseScope: pickers.pickDatabaseScope, + }, ); expect(result.status).toBe('ready'); expect(listSchemas).toHaveBeenCalledWith(tempDir, 'postgres-warehouse'); - expect(prompts.multiselect).toHaveBeenCalledWith({ - message: expect.stringContaining('PostgreSQL schemas to scan'), - options: [ - { value: 'orbit_analytics', label: 'orbit_analytics' }, - { value: 'orbit_raw', label: 'orbit_raw' }, - { value: 'public', label: 'public' }, - ], - initialValues: ['orbit_analytics', 'orbit_raw'], - required: true, + expect(pickers.scopeCalls).toHaveLength(1); + expect(pickers.scopeCalls[0]).toMatchObject({ + connectionId: 'postgres-warehouse', + schemaNoun: 'schema', + schemaNounPlural: 'schemas', + defaultSchemas: ['orbit_analytics', 'orbit_raw'], }); const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); expect(config.connections['postgres-warehouse']).toMatchObject({ diff --git a/packages/cli/src/setup-databases.ts b/packages/cli/src/setup-databases.ts index 5b5b5f8a..c21ab6d1 100644 --- a/packages/cli/src/setup-databases.ts +++ b/packages/cli/src/setup-databases.ts @@ -14,6 +14,11 @@ import { import type { KtxTableListEntry } from '@ktx/context/scan'; import type { KtxCliIo } from './cli-runtime.js'; import { runKtxConnection } from './connection.js'; +import { + pickDatabaseScope as defaultPickDatabaseScope, + type DatabaseScopePickResult, + type PickDatabaseScopeArgs, +} from './database-tree-picker.js'; import { withMultiselectNavigation, withTextInputNavigation } from './prompt-navigation.js'; import { runKtxScan } from './scan.js'; import { writeProjectLocalSecretReference } from './setup-secrets.js'; @@ -90,7 +95,8 @@ export interface KtxSetupDatabasesDeps { scanConnection?: (projectDir: string, connectionId: string, io: KtxCliIo) => Promise; rebuildNativeSqlite?: (io: KtxCliIo) => Promise; listSchemas?: (projectDir: string, connectionId: string) => Promise; - listTables?: (projectDir: string, connectionId: string) => Promise; + listTables?: (projectDir: string, connectionId: string, schemas?: string[]) => Promise; + pickDatabaseScope?: (args: PickDatabaseScopeArgs, io: KtxCliIo) => Promise; historicSqlProbe?: KtxSetupHistoricSqlProbe; } @@ -176,6 +182,7 @@ const SCOPE_DISCOVERY_SPECS: Partial; +type ConnectionSetupStatus = 'ready' | 'back' | 'failed'; const DRIVER_CONNECTION_DEFAULTS: Record = { postgres: { port: '5432' }, @@ -227,6 +234,16 @@ function unique(values: string[]): string[] { return [...new Set(values.filter((value) => value.trim().length > 0))]; } +function stringConfigField(connection: KtxProjectConnectionConfig | undefined, field: string): string | undefined { + const value = connection?.[field]; + return typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined; +} + +function numberConfigField(connection: KtxProjectConnectionConfig | undefined, field: string): number | undefined { + const value = connection?.[field]; + return typeof value === 'number' && Number.isFinite(value) ? value : undefined; +} + function historicSqlConfigRecord(connection: KtxProjectConnectionConfig | undefined): Record | null { const historicSql = connection?.historicSql; return historicSql && typeof historicSql === 'object' && !Array.isArray(historicSql) @@ -352,11 +369,15 @@ function configuredSchemas(connection: KtxProjectConnectionConfig | undefined, d return values.length > 0 ? values : undefined; } -async function defaultListTables(projectDir: string, connectionId: string): Promise { +async function defaultListTables( + projectDir: string, + connectionId: string, + schemasOverride?: string[], +): Promise { const project = await loadKtxProject({ projectDir }); const connection = project.config.connections[connectionId]; const driver = normalizeDriver(connection?.driver); - const schemas = driver ? configuredSchemas(connection, driver) : undefined; + const schemas = schemasOverride ?? (driver ? configuredSchemas(connection, driver) : undefined); if (driver === 'postgres') { const { KtxPostgresScanConnector, isKtxPostgresConnectionConfig } = await import('@ktx/connector-postgres'); @@ -454,6 +475,18 @@ function configuredPrimaryConnectionIds( .sort((left, right) => left.localeCompare(right)); } +function configuredPrimaryDrivers( + connections: Record, + connectionIds: string[], +): KtxSetupDatabaseDriver[] { + const configured = new Set( + connectionIds + .map((connectionId) => normalizeDriver(connections[connectionId]?.driver)) + .filter((driver): driver is KtxSetupDatabaseDriver => driver !== null), + ); + return DRIVER_OPTIONS.map((option) => option.value).filter((driver) => configured.has(driver)); +} + function configuredPrimarySourcesPrompt(connectionIds: string[]): { message: string; options: Array<{ value: string; label: string }>; @@ -462,7 +495,8 @@ function configuredPrimarySourcesPrompt(connectionIds: string[]): { message: `Primary sources already configured: ${connectionIds.join(', ')}\nWhat would you like to do?`, options: [ { value: 'continue', label: 'Continue to knowledge sources' }, - { value: 'add', label: 'Add another primary source' }, + { value: 'edit', label: 'Edit an existing primary source' }, + { value: 'add', label: 'Add additional primary sources' }, ], }; } @@ -552,23 +586,40 @@ async function buildFieldsConnectionConfig(input: { connectionId: string; args: KtxSetupDatabasesArgs; prompts: KtxSetupDatabasesPromptAdapter; + existingConnection?: KtxProjectConnectionConfig; }): Promise { const label = driverLabel(input.driver); const defaults = DRIVER_CONNECTION_DEFAULTS[input.driver]; - const host = await promptText(input.prompts, `${label} host`, 'localhost'); + const host = await promptText( + input.prompts, + `${label} host`, + stringConfigField(input.existingConnection, 'host') ?? 'localhost', + ); if (host === undefined) return 'back'; if (!host) return null; - const portStr = await promptText(input.prompts, `${label} port`, defaults.port); + const portStr = await promptText( + input.prompts, + `${label} port`, + String(numberConfigField(input.existingConnection, 'port') ?? defaults.port), + ); if (portStr === undefined) return 'back'; const port = Number(portStr || defaults.port); - const database = await promptText(input.prompts, `${label} database name`); + const database = await promptText( + input.prompts, + `${label} database name`, + stringConfigField(input.existingConnection, 'database'), + ); if (database === undefined) return 'back'; if (!database) return null; - const username = await promptText(input.prompts, `${label} username`); + const username = await promptText( + input.prompts, + `${label} username`, + stringConfigField(input.existingConnection, 'username'), + ); if (username === undefined) return 'back'; if (!username) return null; @@ -583,6 +634,7 @@ async function buildFieldsConnectionConfig(input: { }); if (credentialResult === 'back') return 'back'; if (credentialResult) passwordRef = credentialResult; + if (!credentialResult) passwordRef = stringConfigField(input.existingConnection, 'password'); } return { @@ -601,9 +653,14 @@ async function buildPastedUrlConnectionConfig(input: { connectionId: string; args: KtxSetupDatabasesArgs; prompts: KtxSetupDatabasesPromptAdapter; + existingConnection?: KtxProjectConnectionConfig; }): Promise { const label = driverLabel(input.driver); - const rawUrl = await promptText(input.prompts, `${label} connection URL`); + const rawUrl = await promptText( + input.prompts, + `${label} connection URL`, + stringConfigField(input.existingConnection, 'url'), + ); if (rawUrl === undefined) return 'back'; if (!rawUrl) return null; @@ -642,6 +699,7 @@ async function buildUrlConnectionConfig(input: { connectionId: string; args: KtxSetupDatabasesArgs; prompts: KtxSetupDatabasesPromptAdapter; + existingConnection?: KtxProjectConnectionConfig; }): Promise { if (input.args.inputMode === 'disabled' && !input.args.databaseUrl) return null; @@ -689,6 +747,7 @@ async function buildConnectionConfig(input: { connectionId: string; args: KtxSetupDatabasesArgs; prompts: KtxSetupDatabasesPromptAdapter; + existingConnection?: KtxProjectConnectionConfig; }): Promise { const { driver, args, prompts } = input; if (driver === 'sqlite') { @@ -698,22 +757,37 @@ async function buildConnectionConfig(input: { (await promptText( prompts, 'SQLite database file\nEnter a relative or absolute path, for example ./warehouse.sqlite.', + stringConfigField(input.existingConnection, 'path'), )); if (path === undefined) return 'back'; return path ? { driver: 'sqlite', path } : null; } if (driver === 'postgres' || driver === 'mysql' || driver === 'clickhouse' || driver === 'sqlserver') { - return await buildUrlConnectionConfig({ driver, connectionId: input.connectionId, args, prompts }); + return await buildUrlConnectionConfig({ + driver, + connectionId: input.connectionId, + args, + prompts, + existingConnection: input.existingConnection, + }); } if (driver === 'bigquery') { - const datasetId = await promptText(prompts, 'BigQuery dataset\nFor example analytics.'); + const datasetId = await promptText( + prompts, + 'BigQuery dataset\nFor example analytics.', + stringConfigField(input.existingConnection, 'dataset_id'), + ); if (datasetId === undefined) return 'back'; - const credentialsPath = await promptText(prompts, 'Path to service account JSON file'); + const credentialsPath = await promptText( + prompts, + 'Path to service account JSON file', + stringConfigField(input.existingConnection, 'credentials_json'), + ); if (credentialsPath === undefined) return 'back'; const location = await promptText( prompts, 'BigQuery location\nPress Enter for US, or enter a location like EU.', - 'US', + stringConfigField(input.existingConnection, 'location') ?? 'US', ); if (location === undefined) return 'back'; if (!datasetId || !credentialsPath) return null; @@ -725,19 +799,35 @@ async function buildConnectionConfig(input: { }; } if (driver === 'snowflake') { - const account = await promptText(prompts, 'Snowflake account identifier'); + const account = await promptText( + prompts, + 'Snowflake account identifier', + stringConfigField(input.existingConnection, 'account'), + ); if (account === undefined) return 'back'; - const warehouse = await promptText(prompts, 'Snowflake warehouse\nFor example ANALYTICS_WH.'); + const warehouse = await promptText( + prompts, + 'Snowflake warehouse\nFor example ANALYTICS_WH.', + stringConfigField(input.existingConnection, 'warehouse'), + ); if (warehouse === undefined) return 'back'; - const database = await promptText(prompts, 'Snowflake database name'); + const database = await promptText( + prompts, + 'Snowflake database name', + stringConfigField(input.existingConnection, 'database'), + ); if (database === undefined) return 'back'; const schemaName = await promptText( prompts, 'Snowflake schema\nPress Enter for PUBLIC, or enter a schema name.', - 'PUBLIC', + stringConfigField(input.existingConnection, 'schema_name') ?? 'PUBLIC', ); if (schemaName === undefined) return 'back'; - const username = await promptText(prompts, 'Snowflake username'); + const username = await promptText( + prompts, + 'Snowflake username', + stringConfigField(input.existingConnection, 'username'), + ); if (username === undefined) return 'back'; const passwordRef = await promptCredential({ prompts, @@ -747,9 +837,14 @@ async function buildConnectionConfig(input: { secretName: 'password', // pragma: allowlist secret }); if (passwordRef === 'back') return 'back'; // pragma: allowlist secret - const role = await promptText(prompts, 'Snowflake role (optional)\nPress Enter to skip.'); + const role = await promptText( + prompts, + 'Snowflake role (optional)\nPress Enter to skip.', + stringConfigField(input.existingConnection, 'role'), + ); if (role === undefined) return 'back'; - if (!account || !warehouse || !database || !schemaName || !username || !passwordRef) return null; + const resolvedPasswordRef = passwordRef ?? stringConfigField(input.existingConnection, 'password'); + if (!account || !warehouse || !database || !schemaName || !username || !resolvedPasswordRef) return null; return { driver: 'snowflake', authMethod: 'password', @@ -758,7 +853,7 @@ async function buildConnectionConfig(input: { database, schema_name: schemaName, username, - password: passwordRef, + password: resolvedPasswordRef, ...(role ? { role } : {}), }; } @@ -1096,6 +1191,59 @@ async function writeConnectionConfig(input: { } } +async function createConnectionConfigRollback(projectDir: string, connectionId: string): Promise<() => Promise> { + const project = await loadKtxProject({ projectDir }); + const previousConnection = project.config.connections[connectionId]; + const hadPreviousConnection = previousConnection !== undefined; + return async () => { + const latest = await loadKtxProject({ projectDir }); + const connections = { ...latest.config.connections }; + if (hadPreviousConnection) { + connections[connectionId] = previousConnection; + } else { + delete connections[connectionId]; + } + await writeFile( + latest.configPath, + serializeKtxProjectConfig({ + ...latest.config, + connections, + }), + 'utf-8', + ); + }; +} + +function withExistingPrimaryEditPromptDefaults(input: { + previous: KtxProjectConnectionConfig; + next: KtxProjectConnectionConfig; + driver: KtxSetupDatabaseDriver; +}): KtxProjectConnectionConfig { + const merged: KtxProjectConnectionConfig = { ...input.next }; + const spec = SCOPE_DISCOVERY_SPECS[input.driver]; + if (spec) { + const nextArray = input.next[spec.configArrayField]; + const previousArray = input.previous[spec.configArrayField]; + if ( + !(Array.isArray(nextArray) && nextArray.length > 0) && + Array.isArray(previousArray) && + previousArray.length > 0 + ) { + delete merged[spec.configSingleField]; + merged[spec.configArrayField] = previousArray; + } else if (!Object.hasOwn(input.next, spec.configArrayField) && !Object.hasOwn(input.next, spec.configSingleField)) { + const previousSingle = input.previous[spec.configSingleField]; + if (typeof previousSingle === 'string' && previousSingle.trim().length > 0) { + merged[spec.configSingleField] = previousSingle; + } + } + } + if (!Object.hasOwn(input.next, 'enabled_tables') && Array.isArray(input.previous.enabled_tables)) { + merged.enabled_tables = input.previous.enabled_tables; + } + return merged; +} + function configuredScopeValues( connection: KtxProjectConnectionConfig | undefined, spec: ScopeDiscoverySpec, @@ -1133,230 +1281,185 @@ async function writeScopeConfig(input: { }); } -async function clearScopeConfig(projectDir: string, connectionId: string): Promise { - const project = await loadKtxProject({ projectDir }); - const connection = project.config.connections[connectionId]; - if (!connection) return; - const driver = normalizeDriver(connection.driver); - if (!driver) return; - const spec = SCOPE_DISCOVERY_SPECS[driver]; - if (!spec) return; - const cleaned = Object.fromEntries( - Object.entries(connection).filter( - ([key]) => key !== spec.configArrayField && key !== spec.configSingleField && key !== 'enabled_tables', - ), - ) as KtxProjectConnectionConfig; - await writeConnectionConfig({ projectDir, connectionId, connection: cleaned }); -} - -async function maybeConfigureSchemaScope(input: { +async function maybeConfigureDatabaseScope(input: { projectDir: string; connectionId: string; args: KtxSetupDatabasesArgs; - prompts: KtxSetupDatabasesPromptAdapter; deps: KtxSetupDatabasesDeps; io: KtxCliIo; -}): Promise { + forcePrompt?: boolean; +}): Promise { const project = await loadKtxProject({ projectDir: input.projectDir }); const connection = project.config.connections[input.connectionId]; const driver = normalizeDriver(connection?.driver); - if (!driver) return true; + if (!driver || driver === 'sqlite') return 'ready'; const spec = SCOPE_DISCOVERY_SPECS[driver]; - if (!spec) return true; + const existingTables = connection?.enabled_tables; + const hasExistingTables = Array.isArray(existingTables) && existingTables.length > 0; + const existingScope = spec ? configuredScopeValues(connection, spec) : []; + const hasExistingScope = !spec || existingScope.length > 0; - const arrayVal = connection?.[spec.configArrayField]; - if (Array.isArray(arrayVal) && arrayVal.length > 0) { - return true; + if (hasExistingTables && hasExistingScope && input.forcePrompt !== true) { + return 'ready'; } - if (input.args.databaseSchemas.length > 0) { + const cliSchemas = input.args.databaseSchemas; + + if (input.args.inputMode === 'disabled') { + if (spec) { + let scopeToWrite: string[] = cliSchemas; + if (scopeToWrite.length === 0) { + try { + scopeToWrite = unique( + await (input.deps.listSchemas ?? defaultListSchemas)(input.projectDir, input.connectionId), + ); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + input.io.stderr.write( + `Could not discover ${spec.promptLabel.toLowerCase()} for ${input.connectionId}; ${detail}\n`, + ); + return 'ready'; + } + } + if (scopeToWrite.length > 0) { + await writeScopeConfig({ + projectDir: input.projectDir, + connectionId: input.connectionId, + values: scopeToWrite, + spec, + }); + const capitalNounPlural = spec.nounPlural[0]!.toUpperCase() + spec.nounPlural.slice(1); + writeSetupSection(input.io, `${capitalNounPlural} saved for ${input.connectionId}`, [ + `✓ ${scopeToWrite.join(', ')}`, + ]); + } + } + return 'ready'; + } + + if (spec && cliSchemas.length > 0) { await writeScopeConfig({ projectDir: input.projectDir, connectionId: input.connectionId, - values: input.args.databaseSchemas, + values: cliSchemas, spec, }); - return true; - } - - writeSetupSection(input.io, `Discovering ${spec.promptLabel.toLowerCase()}`, [ - `Connecting to ${input.connectionId}…`, - ]); - - let discovered: string[]; - try { - discovered = unique( - await (input.deps.listSchemas ?? defaultListSchemas)(input.projectDir, input.connectionId), - ); - } catch (error) { - input.io.stderr.write( - `Could not discover ${spec.promptLabel.toLowerCase()} for ${input.connectionId}; continuing with existing ${spec.noun} scope. ` + - `Pass --database-schema to set it explicitly. ${error instanceof Error ? error.message : String(error)}\n`, - ); - return true; - } - if (discovered.length === 0) { - return true; - } - - let selected: string[]; - if (input.args.inputMode === 'disabled' || discovered.length === 1) { - const preconfigured = configuredScopeValues(connection, spec).filter((v) => discovered.includes(v)); - selected = preconfigured.length > 0 ? preconfigured : discovered; - } else { - const preconfigured = configuredScopeValues(connection, spec).filter((v) => discovered.includes(v)); - const initialValues = preconfigured.length > 0 ? preconfigured : spec.defaultSelection(discovered); - const choices = await input.prompts.multiselect({ - message: withMultiselectNavigation( - `${spec.promptLabel} to scan\n` + - `KTX found multiple ${spec.nounPlural}. Select every ${spec.noun} agents should use.`, - ), - options: discovered.map((v) => ({ value: v, label: v })), - initialValues, - required: true, - }); - if (choices.includes('back')) { - return false; - } - selected = choices.length > 0 ? choices : initialValues; - } - - await writeScopeConfig({ - projectDir: input.projectDir, - connectionId: input.connectionId, - values: selected, - spec, - }); - const capitalNounPlural = spec.nounPlural[0]!.toUpperCase() + spec.nounPlural.slice(1); - writeSetupSection(input.io, `${capitalNounPlural} saved for ${input.connectionId}`, [ - `✓ ${selected.join(', ')}`, - ]); - return true; -} - -async function maybeConfigureTableScope(input: { - projectDir: string; - connectionId: string; - args: KtxSetupDatabasesArgs; - prompts: KtxSetupDatabasesPromptAdapter; - io: KtxCliIo; - deps: KtxSetupDatabasesDeps; -}): Promise { - const project = await loadKtxProject({ projectDir: input.projectDir }); - const connection = project.config.connections[input.connectionId]; - const driver = normalizeDriver(connection?.driver); - if (!driver || driver === 'sqlite') return true; - - const existingTables = connection?.enabled_tables; - if (Array.isArray(existingTables) && existingTables.length > 0) { - return true; - } - - if (input.args.inputMode === 'disabled') { - return true; } writeSetupSection(input.io, 'Discovering tables', [ `Connecting to ${input.connectionId}…`, ]); + const schemasFilter = await (async (): Promise => { + if (cliSchemas.length > 0) return cliSchemas; + if (!spec) return []; + try { + return unique( + await (input.deps.listSchemas ?? defaultListSchemas)(input.projectDir, input.connectionId), + ); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + input.io.stderr.write( + `Could not discover ${spec.promptLabel.toLowerCase()} for ${input.connectionId}; ${detail}\n`, + ); + return []; + } + })(); + let discovered: KtxTableListEntry[]; try { discovered = await (input.deps.listTables ?? defaultListTables)( input.projectDir, input.connectionId, + schemasFilter.length > 0 ? schemasFilter : undefined, ); } catch (error) { + const detail = error instanceof Error ? error.message : String(error); input.io.stderr.write( - `Could not discover tables for ${input.connectionId}; continuing without table filter. ` + - `${error instanceof Error ? error.message : String(error)}\n`, + input.forcePrompt === true + ? `Could not discover tables for ${input.connectionId}; edit was not saved. ${detail}\n` + : `Could not discover tables for ${input.connectionId}; continuing without table filter. ${detail}\n`, ); - return true; + return input.forcePrompt === true ? 'failed' : 'ready'; } if (discovered.length === 0) { - return true; + if (input.forcePrompt === true) { + input.io.stderr.write(`No tables discovered for ${input.connectionId}; edit was not saved.\n`); + } + return input.forcePrompt === true ? 'failed' : 'ready'; } const allQualified = discovered.map((t) => `${t.schema}.${t.name}`); + const schemasInDiscovery = unique(discovered.map((t) => t.schema)); + + const defaultSchemas = (() => { + if (cliSchemas.length > 0) return cliSchemas; + if (!spec) return schemasInDiscovery; + return spec.defaultSelection(schemasInDiscovery); + })(); + + const existingEnabled = + hasExistingTables && input.forcePrompt === true + ? (existingTables ?? []).filter( + (table): table is string => typeof table === 'string' && allQualified.includes(table), + ) + : []; + + let activeSchemas: string[]; + let enabledTables: string[]; if (discovered.length === 1) { - await writeConnectionConfig({ + enabledTables = allQualified; + activeSchemas = spec ? schemasInDiscovery : []; + } else { + const pickResult = await (input.deps.pickDatabaseScope ?? defaultPickDatabaseScope)( + { + connectionId: input.connectionId, + schemaNoun: spec?.noun ?? 'schema', + schemaNounPlural: spec?.nounPlural ?? 'schemas', + discovered, + existing: { enabledTables: existingEnabled }, + defaultSchemas, + supportsSchemaScope: spec !== undefined, + }, + input.io, + ); + if (pickResult.kind === 'back') { + return 'back'; + } + enabledTables = pickResult.enabledTables; + activeSchemas = pickResult.activeSchemas; + } + + if (spec) { + await writeScopeConfig({ projectDir: input.projectDir, connectionId: input.connectionId, - connection: { ...connection!, enabled_tables: allQualified }, + values: activeSchemas, + spec, }); - writeSetupSection(input.io, `Tables enabled for ${input.connectionId}`, [ - `✓ ${allQualified[0]}`, - ]); - return true; } - - const bySchema = new Map(); - for (const entry of discovered) { - const existing = bySchema.get(entry.schema) ?? []; - existing.push(entry); - bySchema.set(entry.schema, existing); - } - const schemaList = [...bySchema.keys()].sort(); - const schemaSummary = schemaList.map((s) => `${s} (${bySchema.get(s)!.length})`).join(', '); - - let selected: string[] | null = null; - - while (selected === null) { - const action = await input.prompts.select({ - message: `Tables found in selected schemas\n` + - `${discovered.length} tables across ${schemaList.length} ${schemaList.length === 1 ? 'schema' : 'schemas'}: ${schemaSummary}`, - options: [ - { value: 'all', label: 'Enable all tables' }, - { value: 'customize', label: 'Customize which tables to enable' }, - { value: 'back', label: 'Back' }, - ], - }); - - if (action === 'back') { - return false; - } - - if (action === 'all') { - selected = allQualified; - } else { - const choices = await input.prompts.multiselect({ - message: withMultiselectNavigation( - `Tables to enable for ${input.connectionId}\n` + - `Deselect any tables agents should not use.`, - ), - options: discovered.map((t) => { - const qualified = `${t.schema}.${t.name}`; - const suffix = t.kind === 'view' ? ' (view)' : ''; - return { value: qualified, label: `${qualified}${suffix}` }; - }), - initialValues: allQualified, - required: true, - }); - - if (choices.includes('back')) { - continue; - } - if (choices.length === 0) { - input.io.stdout.write('│ KTX needs at least one table enabled. Select a table or press Escape to go back.\n'); - continue; - } - selected = choices; - } - } - + const refreshedProject = await loadKtxProject({ projectDir: input.projectDir }); + const currentConnection = refreshedProject.config.connections[input.connectionId]; + if (!currentConnection) return 'ready'; await writeConnectionConfig({ projectDir: input.projectDir, connectionId: input.connectionId, - connection: { ...connection!, enabled_tables: selected }, + connection: { ...currentConnection, enabled_tables: enabledTables }, }); + if (spec && activeSchemas.length > 0) { + const capitalNounPlural = spec.nounPlural[0]!.toUpperCase() + spec.nounPlural.slice(1); + writeSetupSection(input.io, `${capitalNounPlural} saved for ${input.connectionId}`, [ + `✓ ${activeSchemas.join(', ')}`, + ]); + } writeSetupSection(input.io, `Tables enabled for ${input.connectionId}`, [ - `✓ ${selected.length}/${discovered.length} tables enabled`, + `✓ ${enabledTables.length}/${discovered.length} tables enabled`, ]); - return true; + return 'ready'; } async function ensureHistoricSqlIngestDefaults(projectDir: string): Promise { @@ -1466,7 +1569,8 @@ async function validateAndScanConnection(input: { deps: KtxSetupDatabasesDeps; args: KtxSetupDatabasesArgs; prompts: KtxSetupDatabasesPromptAdapter; -}): Promise { + forceScopeAndTables?: boolean; +}): Promise { const testConnection = input.deps.testConnection ?? defaultTestConnection; const scanConnection = input.deps.scanConnection ?? defaultScanConnection; const project = await loadKtxProject({ projectDir: input.projectDir }); @@ -1477,7 +1581,7 @@ async function validateAndScanConnection(input: { if (testCode !== 0) { flushBufferedCommandOutput(input.io, testIo); input.io.stderr.write(`Connection test failed for ${input.connectionId}.\n`); - return false; + return 'failed'; } const testOutput = testIo.stdoutText(); const outputDriver = normalizeDriver(readOutputValue(testOutput, 'Driver')); @@ -1485,16 +1589,9 @@ async function validateAndScanConnection(input: { const testLines = ['✓ Connection test passed', `Driver: ${driverDisplay}`]; writeSetupSection(input.io, `Testing ${input.connectionId}`, testLines); - while (true) { - if (!(await maybeConfigureSchemaScope(input))) { - return false; - } - - if (await maybeConfigureTableScope(input)) { - break; - } - - await clearScopeConfig(input.projectDir, input.connectionId); + const scopeStatus = await maybeConfigureDatabaseScope({ ...input, forcePrompt: input.forceScopeAndTables }); + if (scopeStatus !== 'ready') { + return scopeStatus; } await maybeRunHistoricSqlSetupProbe({ @@ -1554,7 +1651,7 @@ async function validateAndScanConnection(input: { ); } if (scanCode !== 0) { - return false; + return 'failed'; } } const scanOutput = scanIo.stdoutText(); @@ -1570,14 +1667,14 @@ async function validateAndScanConnection(input: { writeSetupSection(input.io, 'Primary source ready', [ `${input.connectionId} · ${driverDisplay} · structural scan complete`, ]); - return true; + return 'ready'; } async function chooseDrivers( args: KtxSetupDatabasesArgs, io: KtxCliIo, prompts: KtxSetupDatabasesPromptAdapter, - options?: { hasPrimarySources?: boolean }, + options?: { hasPrimarySources?: boolean; initialDrivers?: KtxSetupDatabaseDriver[] }, ): Promise { if (args.databaseDrivers && args.databaseDrivers.length > 0) { return [...new Set(args.databaseDrivers)]; @@ -1592,10 +1689,12 @@ async function chooseDrivers( return 'missing-input'; } while (true) { + const initialValues = unique(options?.initialDrivers ?? []); const choices = await prompts.multiselect({ message: withMultiselectNavigation('Which primary sources should KTX connect to?'), options: [...DRIVER_OPTIONS], - required: false, + ...(initialValues.length > 0 ? { initialValues } : {}), + required: options?.hasPrimarySources === true, }); if (choices.includes('back')) { return 'back'; @@ -1617,7 +1716,7 @@ async function chooseConnectionIdForDriver(input: { connections: Record; args: KtxSetupDatabasesArgs; prompts: KtxSetupDatabasesPromptAdapter; -}): Promise<{ kind: 'existing' | 'new'; connectionId: string } | 'back' | 'missing-input'> { +}): Promise<{ kind: 'existing' | 'new' | 'edit'; connectionId: string } | 'back' | 'missing-input'> { if (input.args.databaseConnectionId) { return { kind: 'new', connectionId: input.args.databaseConnectionId }; } @@ -1647,14 +1746,19 @@ async function chooseConnectionIdForDriver(input: { options: [ ...existingIds.map((connectionId) => ({ value: `existing:${connectionId}`, - label: `Use existing ${label} connection: ${connectionId}`, + label: `Keep existing ${label} connection: ${connectionId}`, })), - { value: 'new', label: `Add new ${label} connection` }, + ...existingIds.map((connectionId) => ({ + value: `edit:${connectionId}`, + label: `Edit ${label} connection: ${connectionId}`, + })), + { value: 'new', label: `Add another ${label} connection` }, { value: 'back', label: 'Back' }, ], }); if (choice === 'back') return 'back'; if (choice.startsWith('existing:')) return { kind: 'existing', connectionId: choice.slice('existing:'.length) }; + if (choice.startsWith('edit:')) return { kind: 'edit', connectionId: choice.slice('edit:'.length) }; const entered = await input.prompts.text({ message: withTextInputNavigation(connectionNamePrompt(label)), placeholder: defaultId, @@ -1666,6 +1770,102 @@ async function chooseConnectionIdForDriver(input: { } } +async function choosePrimarySourceToEdit(input: { + projectDir: string; + connectionIds: string[]; + prompts: KtxSetupDatabasesPromptAdapter; +}): Promise { + const project = await loadKtxProject({ projectDir: input.projectDir }); + const options = input.connectionIds + .map((connectionId) => { + const driver = normalizeDriver(project.config.connections[connectionId]?.driver); + if (!driver) return null; + return { value: connectionId, label: `${connectionId} (${driverLabel(driver)})` }; + }) + .filter((option): option is { value: string; label: string } => option !== null); + if (options.length === 0) return 'back'; + const choice = await input.prompts.select({ + message: 'Primary source to edit', + options: [...options, { value: 'back', label: 'Back' }], + }); + return choice === 'back' ? 'back' : choice; +} + +async function runPrimarySourceFullEdit(input: { + projectDir: string; + connectionId: string; + args: KtxSetupDatabasesArgs; + prompts: KtxSetupDatabasesPromptAdapter; + io: KtxCliIo; + deps: KtxSetupDatabasesDeps; +}): Promise<'ready' | 'back' | 'failed'> { + const project = await loadKtxProject({ projectDir: input.projectDir }); + const existing = project.config.connections[input.connectionId]; + const driver = normalizeDriver(existing?.driver); + if (!existing || !driver) { + input.io.stderr.write(`Connection "${input.connectionId}" is not a configured primary source.\n`); + return 'failed'; + } + + const rollback = await createConnectionConfigRollback(input.projectDir, input.connectionId); + const replacement = await buildConnectionConfig({ + driver, + connectionId: input.connectionId, + args: input.args, + prompts: input.prompts, + existingConnection: existing, + }); + if (replacement === 'back') { + await rollback(); + return 'back'; + } + if (!replacement) { + await rollback(); + return 'failed'; + } + + const withHistoricSql = await maybeApplyHistoricSqlConfig({ + connection: replacement, + driver, + args: input.args, + prompts: input.prompts, + }); + if (withHistoricSql === 'back') { + await rollback(); + return 'back'; + } + + await writeConnectionConfig({ + projectDir: input.projectDir, + connectionId: input.connectionId, + connection: withExistingPrimaryEditPromptDefaults({ + previous: existing, + next: { + ...withHistoricSql, + ...(!Object.hasOwn(withHistoricSql, 'historicSql') && existing.historicSql !== undefined + ? { historicSql: existing.historicSql } + : {}), + }, + driver, + }), + }); + + const validated = await validateAndScanConnection({ + projectDir: input.projectDir, + connectionId: input.connectionId, + io: input.io, + deps: input.deps, + args: input.args, + prompts: input.prompts, + forceScopeAndTables: true, + }); + if (validated !== 'ready') { + await rollback(); + return validated; + } + return 'ready'; +} + export async function runKtxSetupDatabasesStep( args: KtxSetupDatabasesArgs, io: KtxCliIo, @@ -1688,7 +1888,18 @@ export async function runKtxSetupDatabasesStep( prompts, }); if (historicSqlResult === 'back') return { status: 'back', projectDir: args.projectDir }; - if (!(await validateAndScanConnection({ projectDir: args.projectDir, connectionId, io, deps, args, prompts }))) { + const setupStatus = await validateAndScanConnection({ + projectDir: args.projectDir, + connectionId, + io, + deps, + args, + prompts, + }); + if (setupStatus === 'back') { + return { status: 'back', projectDir: args.projectDir }; + } + if (setupStatus === 'failed') { return { status: 'failed', projectDir: args.projectDir }; } selectedConnectionIds.push(connectionId); @@ -1712,10 +1923,43 @@ export async function runKtxSetupDatabasesStep( await markDatabasesComplete(args.projectDir, selectedConnectionIds); return { status: 'ready', projectDir: args.projectDir, connectionIds: selectedConnectionIds }; } + if (action === 'edit') { + const connectionId = await choosePrimarySourceToEdit({ + projectDir: args.projectDir, + connectionIds: selectedConnectionIds, + prompts, + }); + if (connectionId === 'back') { + showConfiguredPrimaryMenu = true; + continue; + } + const editResult = await runPrimarySourceFullEdit({ + projectDir: args.projectDir, + connectionId, + args, + prompts, + io, + deps, + }); + if (editResult === 'back') { + showConfiguredPrimaryMenu = true; + continue; + } + if (editResult === 'failed') { + return { status: 'failed', projectDir: args.projectDir }; + } + pushUniqueConnectionId(selectedConnectionIds, connectionId); + showConfiguredPrimaryMenu = true; + continue; + } } showConfiguredPrimaryMenu = false; - const drivers = await chooseDrivers(args, io, prompts, { hasPrimarySources: selectedConnectionIds.length > 0 }); + const driverProject = await loadKtxProject({ projectDir: args.projectDir }); + const drivers = await chooseDrivers(args, io, prompts, { + hasPrimarySources: selectedConnectionIds.length > 0, + initialDrivers: configuredPrimaryDrivers(driverProject.config.connections, selectedConnectionIds), + }); if (drivers === 'back') { if (selectedConnectionIds.length > 0 && canReturnToDriverSelection && args.inputMode !== 'disabled') { showConfiguredPrimaryMenu = true; @@ -1750,7 +1994,26 @@ export async function runKtxSetupDatabasesStep( return { status: 'missing-input', projectDir: args.projectDir }; } - if (connectionChoice.kind === 'new') { + let connectionAlreadyValidated = false; + if (connectionChoice.kind === 'edit') { + const editResult = await runPrimarySourceFullEdit({ + projectDir: args.projectDir, + connectionId: connectionChoice.connectionId, + args, + prompts, + io, + deps, + }); + if (editResult === 'back') { + if (!canReturnToDriverSelection) return { status: 'back', projectDir: args.projectDir }; + returnToDriverSelection = true; + break; + } + if (editResult === 'failed') { + return { status: 'failed', projectDir: args.projectDir }; + } + connectionAlreadyValidated = true; + } else if (connectionChoice.kind === 'new') { let connection = await buildConnectionConfig({ driver, connectionId: connectionChoice.connectionId, @@ -1819,16 +2082,22 @@ export async function runKtxSetupDatabasesStep( } let connectionSkipped = false; - while ( - !(await validateAndScanConnection({ - projectDir: args.projectDir, - connectionId: connectionChoice.connectionId, - io, - deps, - args, - prompts, - })) - ) { + let setupStatus: ConnectionSetupStatus = connectionAlreadyValidated + ? 'ready' + : await validateAndScanConnection({ + projectDir: args.projectDir, + connectionId: connectionChoice.connectionId, + io, + deps, + args, + prompts, + }); + while (!connectionAlreadyValidated && setupStatus !== 'ready') { + if (setupStatus === 'back') { + if (!canReturnToDriverSelection) return { status: 'back', projectDir: args.projectDir }; + returnToDriverSelection = true; + break; + } if (args.inputMode === 'disabled') return { status: 'failed', projectDir: args.projectDir }; const action = await prompts.select({ message: `Primary source setup failed for ${connectionChoice.connectionId}`, @@ -1848,7 +2117,16 @@ export async function runKtxSetupDatabasesStep( connectionSkipped = true; break; } - if (action === 're-enter') { + if (action === 'retry') { + setupStatus = await validateAndScanConnection({ + projectDir: args.projectDir, + connectionId: connectionChoice.connectionId, + io, + deps, + args, + prompts, + }); + } else if (action === 're-enter') { const connection = await buildConnectionConfig({ driver, connectionId: connectionChoice.connectionId, @@ -1872,6 +2150,14 @@ export async function runKtxSetupDatabasesStep( connectionId: connectionChoice.connectionId, connection: withHistoricSql, }); + setupStatus = await validateAndScanConnection({ + projectDir: args.projectDir, + connectionId: connectionChoice.connectionId, + io, + deps, + args, + prompts, + }); } } if (returnToDriverSelection) break; diff --git a/packages/cli/src/setup-embeddings.test.ts b/packages/cli/src/setup-embeddings.test.ts index c2d5cad2..4c91ddd5 100644 --- a/packages/cli/src/setup-embeddings.test.ts +++ b/packages/cli/src/setup-embeddings.test.ts @@ -319,14 +319,14 @@ describe('setup embeddings step', () => { projectDir: tempDir, inputMode: 'disabled', embeddingBackend: 'openai', - embeddingApiKeyEnv: 'OPENAI_API_KEY', + embeddingApiKeyEnv: 'OPENAI_API_KEY', // pragma: allowlist secret cliVersion: '0.2.0', runtimeInstallPolicy: 'auto', skipEmbeddings: false, }, io.io, { - env: { OPENAI_API_KEY: 'sk-openai-test' }, + env: { OPENAI_API_KEY: 'sk-openai-test' }, // pragma: allowlist secret healthCheck, }, ); @@ -336,14 +336,14 @@ describe('setup embeddings step', () => { backend: 'openai', model: 'text-embedding-3-small', dimensions: 1536, - openai: { apiKey: 'sk-openai-test' }, + openai: { apiKey: 'sk-openai-test' }, // pragma: allowlist secret }); const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); expect(config.ingest.embeddings).toMatchObject({ backend: 'openai', model: 'text-embedding-3-small', dimensions: 1536, - openai: { api_key: 'env:OPENAI_API_KEY' }, + openai: { api_key: 'env:OPENAI_API_KEY' }, // pragma: allowlist secret }); expect(io.stdout()).not.toContain('sk-openai-test'); }); @@ -367,7 +367,7 @@ describe('setup embeddings step', () => { io.io, { prompts, - env: { OPENAI_API_KEY: 'sk-openai-test' }, + env: { OPENAI_API_KEY: 'sk-openai-test' }, // pragma: allowlist secret healthCheck, ensureLocalEmbeddings: vi.fn(async () => managedDaemon()), }, @@ -384,7 +384,7 @@ describe('setup embeddings step', () => { backend: 'openai', model: 'text-embedding-3-small', dimensions: 1536, - openai: { apiKey: 'sk-openai-test' }, + openai: { apiKey: 'sk-openai-test' }, // pragma: allowlist secret }); expect(prompts.select).toHaveBeenCalledWith( expect.objectContaining({ @@ -478,7 +478,7 @@ describe('setup embeddings step', () => { }, makeIo().io, { - env: { OPENAI_API_KEY: 'sk-openai-test' }, + env: { OPENAI_API_KEY: 'sk-openai-test' }, // pragma: allowlist secret healthCheck, }, ), diff --git a/packages/cli/src/setup-models.test.ts b/packages/cli/src/setup-models.test.ts index e310ea90..e4425d69 100644 --- a/packages/cli/src/setup-models.test.ts +++ b/packages/cli/src/setup-models.test.ts @@ -7,10 +7,8 @@ import { BUNDLED_ANTHROPIC_MODELS, fetchAnthropicModels, type KtxSetupModelPromptAdapter, - runKtxSetupGcloudApplicationDefaultAuth, runKtxSetupAnthropicModelStep, } from './setup-models.js'; -import type { KtxCliIo } from './cli-runtime.js'; function makeIo() { let stdout = ''; @@ -34,6 +32,17 @@ function makeIo() { }; } +function makeSpinnerEvents() { + const events: string[] = []; + const spinner = vi.fn(() => ({ + start: (msg: string) => events.push(`start:${msg}`), + message: (msg: string) => events.push(`message:${msg}`), + stop: (msg: string) => events.push(`stop:${msg}`), + error: (msg: string) => events.push(`error:${msg}`), + })); + return { events, spinner }; +} + function makePromptAdapter(options: { providerChoice?: string; selectValues?: string[]; @@ -191,6 +200,7 @@ describe('setup Anthropic model step', () => { it('configures env credentials, selected model, prompt caching, and llm completion state', async () => { const io = makeIo(); + const { events: spinnerEvents, spinner } = makeSpinnerEvents(); const result = await runKtxSetupAnthropicModelStep( { projectDir: tempDir, @@ -203,6 +213,7 @@ describe('setup Anthropic model step', () => { { env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, // pragma: allowlist secret healthCheck: vi.fn(async () => ({ ok: true as const })), + spinner, }, ); @@ -219,6 +230,10 @@ describe('setup Anthropic model step', () => { expect(config.scan.enrichment.mode).toBe('llm'); expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:'); expect((await readKtxSetupState(tempDir)).completed_steps).toContain('llm'); + expect(spinnerEvents).toEqual([ + 'start:Checking Anthropic API LLM (claude-sonnet-4-6).', + 'stop:LLM test passed (Anthropic API, claude-sonnet-4-6)', + ]); expect(io.stdout()).toContain('LLM ready: yes'); expect(io.stdout()).not.toContain('sk-ant-test'); }); @@ -226,6 +241,7 @@ describe('setup Anthropic model step', () => { it('configures Vertex AI provider, selected model, prompt caching, and llm completion state', async () => { const io = makeIo(); const healthCheck = vi.fn(async () => ({ ok: true as const })); + const { events: spinnerEvents, spinner } = makeSpinnerEvents(); const result = await runKtxSetupAnthropicModelStep( { @@ -238,7 +254,7 @@ describe('setup Anthropic model step', () => { skipLlm: false, }, io.io, - { env: {}, healthCheck }, + { env: {}, healthCheck, spinner }, ); expect(result.status).toBe('ready'); @@ -260,13 +276,16 @@ describe('setup Anthropic model step', () => { expect(config.scan.enrichment.mode).toBe('llm'); expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:'); expect((await readKtxSetupState(tempDir)).completed_steps).toContain('llm'); + expect(spinnerEvents).toEqual([ + 'start:Checking Vertex AI LLM (claude-sonnet-4-6).', + 'stop:LLM test passed (Vertex AI, claude-sonnet-4-6)', + ]); expect(io.stdout()).toContain('LLM ready: yes (claude-sonnet-4-6)'); }); - it('can run gcloud auth for Vertex AI and infer project and default location', async () => { + it('uses existing Vertex AI credentials without offering to run gcloud auth', async () => { const io = makeIo(); - const prompts = makePromptAdapter({ selectValues: ['vertex', 'gcloud', 'local-gcp-project', 'claude-sonnet-4-6'] }); - const runGcloudAuth = vi.fn(async () => ({ ok: true as const })); + const prompts = makePromptAdapter({ selectValues: ['vertex', 'existing', 'local-gcp-project', 'claude-sonnet-4-6'] }); const readGcloudProject = vi.fn(async () => 'local-gcp-project'); const listGcloudProjects = vi.fn(async () => [ { projectId: 'local-gcp-project', name: 'Local project' }, @@ -280,7 +299,6 @@ describe('setup Anthropic model step', () => { { prompts, env: {}, - runGcloudAuth, readGcloudProject, listGcloudProjects, healthCheck, @@ -288,7 +306,15 @@ describe('setup Anthropic model step', () => { ); expect(result.status).toBe('ready'); - expect(runGcloudAuth).toHaveBeenCalledWith(io.io); + expect(prompts.select).toHaveBeenCalledWith( + expect.objectContaining({ + message: expect.stringContaining('How should KTX authenticate with Google Vertex AI?'), + options: [ + { value: 'existing', label: 'Use existing gcloud/Application Default Credentials' }, + { value: 'back', label: 'Back' }, + ], + }), + ); expect(readGcloudProject).toHaveBeenCalled(); expect(listGcloudProjects).toHaveBeenCalled(); expect(prompts.text).not.toHaveBeenCalled(); @@ -303,6 +329,22 @@ describe('setup Anthropic model step', () => { ], }), ); + expect(prompts.select).toHaveBeenCalledWith( + expect.objectContaining({ + message: expect.stringContaining('Which Anthropic model should KTX use?'), + options: [ + { value: 'claude-opus-4-7', label: 'Claude Opus 4.7' }, + { value: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6' }, + { value: 'claude-opus-4-6', label: 'Claude Opus 4.6' }, + { value: 'claude-opus-4-5', label: 'Claude Opus 4.5' }, + { value: 'claude-haiku-4-5', label: 'Claude Haiku 4.5' }, + { value: 'claude-sonnet-4-5', label: 'Claude Sonnet 4.5' }, + { value: 'claude-opus-4-1', label: 'Claude Opus 4.1' }, + { value: 'manual', label: 'Enter a model ID manually' }, + { value: 'back', label: 'Back' }, + ], + }), + ); expect(healthCheck).toHaveBeenCalledWith({ backend: 'vertex', vertex: { project: 'local-gcp-project', location: 'us-east5' }, @@ -415,35 +457,6 @@ describe('setup Anthropic model step', () => { ); }); - it('runs only gcloud application-default login for Vertex AI auth', async () => { - const io = makeIo(); - const runGcloud = vi.fn(async () => ({ ok: true as const })); - - await expect(runKtxSetupGcloudApplicationDefaultAuth(io.io, runGcloud)).resolves.toEqual({ ok: true }); - - expect(runGcloud).toHaveBeenCalledTimes(1); - expect(runGcloud).toHaveBeenCalledWith(['auth', 'application-default', 'login'], expect.anything()); - expect(runGcloud).not.toHaveBeenCalledWith(['auth', 'login'], expect.anything()); - expect(io.stdout()).toContain('gcloud auth application-default login'); - expect(io.stdout()).not.toContain('gcloud auth login'); - }); - - it('indents gcloud auth output inside the setup gutter', async () => { - const io = makeIo(); - const runGcloud = vi.fn(async (_args: string[], commandIo: KtxCliIo) => { - commandIo.stdout.write('Your browser has been opened to visit:\n\n https://accounts.example/auth\n'); - commandIo.stderr.write('Credentials saved to file: [/tmp/application_default_credentials.json]\n'); - return { ok: true as const }; - }); - - await expect(runKtxSetupGcloudApplicationDefaultAuth(io.io, runGcloud)).resolves.toEqual({ ok: true }); - - expect(io.stdout()).toContain('│ Your browser has been opened to visit:'); - expect(io.stdout()).toContain('│ https://accounts.example/auth'); - expect(io.stderr()).toContain('│ Credentials saved to file: [/tmp/application_default_credentials.json]'); - expect(io.stdout()).not.toContain('\nYour browser has been opened'); - }); - it('explains common Vertex AI Forbidden health-check causes', async () => { const io = makeIo(); diff --git a/packages/cli/src/setup-models.ts b/packages/cli/src/setup-models.ts index bd05bd44..e4c7fcd2 100644 --- a/packages/cli/src/setup-models.ts +++ b/packages/cli/src/setup-models.ts @@ -1,4 +1,4 @@ -import { execFile, spawn } from 'node:child_process'; +import { execFile } from 'node:child_process'; import { writeFile } from 'node:fs/promises'; import { promisify } from 'node:util'; import { resolveLocalKtxLlmConfig } from '@ktx/context'; @@ -11,6 +11,7 @@ import { serializeKtxProjectConfig, } from '@ktx/context/project'; import { type KtxLlmConfig, type KtxLlmHealthCheckResult, runKtxLlmHealthCheck } from '@ktx/llm'; +import { createClackSpinner, type KtxCliSpinner } from './clack.js'; import type { KtxCliIo } from './cli-runtime.js'; import { withTextInputNavigation } from './prompt-navigation.js'; import { envCredentialReference, writeProjectLocalSecretReference } from './setup-secrets.js'; @@ -61,9 +62,9 @@ export interface KtxSetupModelDeps { prompts?: KtxSetupModelPromptAdapter; listModels?: (apiKey: string) => Promise; healthCheck?: (config: KtxLlmConfig) => Promise; - runGcloudAuth?: (io: KtxCliIo) => Promise; readGcloudProject?: () => Promise; listGcloudProjects?: () => Promise; + spinner?: () => KtxCliSpinner; } export const BUNDLED_ANTHROPIC_MODEL_REGISTRY_VERSION = '2026-05-07'; @@ -74,6 +75,16 @@ export const BUNDLED_ANTHROPIC_MODELS: AnthropicModelChoice[] = [ { id: 'claude-haiku-4-5', label: 'Claude Haiku 4.5', recommended: false }, ]; +const VERTEX_ANTHROPIC_MODELS: AnthropicModelChoice[] = [ + { id: 'claude-opus-4-7', label: 'Claude Opus 4.7', recommended: false }, + { id: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6', recommended: false }, + { id: 'claude-opus-4-6', label: 'Claude Opus 4.6', recommended: false }, + { id: 'claude-opus-4-5', label: 'Claude Opus 4.5', recommended: false }, + { id: 'claude-haiku-4-5', label: 'Claude Haiku 4.5', recommended: false }, + { id: 'claude-sonnet-4-5', label: 'Claude Sonnet 4.5', recommended: false }, + { id: 'claude-opus-4-1', label: 'Claude Opus 4.1', recommended: false }, +]; + const HIDDEN_ANTHROPIC_MODEL_PATTERNS = [ /^claude-sonnet-4$/i, /^claude-opus-4$/i, @@ -91,8 +102,8 @@ const ANTHROPIC_MODEL_PROMPT_CONTEXT = 'into semantic-layer sources and wiki context.'; const VERTEX_AUTH_PROMPT_CONTEXT = - 'KTX can use Google Cloud Application Default Credentials for local Vertex AI access. This opens the normal ' + - 'gcloud browser login flow and does not store Google credentials in ktx.yaml.'; + 'KTX uses Google Cloud Application Default Credentials for local Vertex AI access and does not store Google ' + + 'credentials in ktx.yaml. If needed, run gcloud auth application-default login before continuing.'; const VERTEX_PROJECT_PROMPT_CONTEXT = 'KTX stores the selected Google Cloud project ID in ktx.yaml and uses Application Default Credentials for ' + 'access. Project visibility depends on the signed-in Google account and organization permissions.'; @@ -137,94 +148,17 @@ type VertexConfigChoice = } | { status: 'back' | 'missing-input' }; -type VertexAuthChoice = { status: 'ready' } | { status: 'back' | 'missing-input' }; +type VertexAuthChoice = { status: 'ready' } | { status: 'back' }; -export type GcloudAuthResult = { ok: true } | { ok: false; message: string }; interface GcloudProjectChoice { projectId: string; name?: string; } -type GcloudCommandRunner = (args: string[], io: KtxCliIo) => Promise; function createPromptAdapter(): KtxSetupModelPromptAdapter { return createKtxSetupPromptAdapter({ selectCancelValue: 'back' }); } -function createIndentedCommandIo(io: KtxCliIo): KtxCliIo { - const indentedWriter = (write: (chunk: string) => void) => { - let atLineStart = true; - return (chunk: string) => { - for (const char of chunk) { - if (atLineStart) { - write('│ '); - atLineStart = false; - } - write(char); - if (char === '\n') { - atLineStart = true; - } - } - }; - }; - - return { - stdout: { - isTTY: io.stdout.isTTY, - columns: io.stdout.columns, - write: indentedWriter((chunk) => io.stdout.write(chunk)), - }, - stderr: { - write: indentedWriter((chunk) => io.stderr.write(chunk)), - }, - }; -} - -function runInteractiveGcloud(args: string[], io: KtxCliIo): Promise { - return new Promise((resolve) => { - let settled = false; - const child = spawn('gcloud', args, { stdio: ['inherit', 'pipe', 'pipe'] }); - child.stdout?.on('data', (chunk: Buffer) => { - io.stdout.write(chunk.toString('utf8')); - }); - child.stderr?.on('data', (chunk: Buffer) => { - io.stderr.write(chunk.toString('utf8')); - }); - child.on('error', (error: NodeJS.ErrnoException) => { - if (settled) { - return; - } - settled = true; - if (error.code === 'ENOENT') { - resolve({ ok: false, message: 'gcloud CLI was not found on PATH.' }); - return; - } - resolve({ ok: false, message: error.message }); - }); - child.on('close', (code, signal) => { - if (settled) { - return; - } - settled = true; - if (code === 0) { - resolve({ ok: true }); - return; - } - resolve({ - ok: false, - message: signal ? `gcloud exited after signal ${signal}.` : `gcloud exited with code ${code ?? 'unknown'}.`, - }); - }); - }); -} - -export async function runKtxSetupGcloudApplicationDefaultAuth( - io: KtxCliIo, - runGcloud: GcloudCommandRunner = runInteractiveGcloud, -): Promise { - io.stdout.write('│ Running gcloud auth application-default login...\n'); - return await runGcloud(['auth', 'application-default', 'login'], createIndentedCommandIo(io)); -} - async function defaultReadGcloudProject(): Promise { try { const { stdout } = await execFileAsync('gcloud', ['config', 'get-value', 'project'], { encoding: 'utf8' }); @@ -374,6 +308,53 @@ function buildVertexHealthConfig(vertex: { project?: string; location: string }, }; } +type LlmHealthProvider = 'Anthropic API' | 'Vertex AI'; + +function llmHealthCheckStartText(provider: LlmHealthProvider, model: string): string { + return `Checking ${provider} LLM (${model}).`; +} + +function startLlmHealthCheckProgress( + spinner: KtxCliSpinner, + message: string, +): { succeed(msg: string): void; fail(msg: string): void } { + spinner.start(message); + return { + succeed(msg: string) { + spinner.stop(msg); + }, + fail(msg: string) { + spinner.error(msg); + }, + }; +} + +async function runLlmHealthCheckWithProgress( + config: KtxLlmConfig, + provider: LlmHealthProvider, + model: string, + healthCheck: (config: KtxLlmConfig) => Promise, + deps: KtxSetupModelDeps, +): Promise { + const progress = startLlmHealthCheckProgress( + (deps.spinner ?? createClackSpinner)(), + llmHealthCheckStartText(provider, model), + ); + let health: KtxLlmHealthCheckResult; + try { + health = await healthCheck(config); + } catch (error) { + progress.fail('LLM test failed'); + throw error; + } + if (health.ok) { + progress.succeed(`LLM test passed (${provider}, ${model})`); + } else { + progress.fail('LLM test failed'); + } + return health; +} + function formatVertexHealthFailure(message: string, vertex: { project?: string; location: string }): string { const trimmed = message.trim() || 'unknown error'; if (!/(forbidden|permission|permission_denied|403)/i.test(trimmed)) { @@ -516,7 +497,6 @@ async function chooseBackend( async function chooseVertexAuth( args: KtxSetupModelArgs, - io: KtxCliIo, deps: KtxSetupModelDeps, ): Promise { if (args.inputMode === 'disabled' || args.vertexProject || args.vertexLocation) { @@ -527,7 +507,6 @@ async function chooseVertexAuth( const choice = await prompts.select({ message: `How should KTX authenticate with Google Vertex AI?\n\n${VERTEX_AUTH_PROMPT_CONTEXT}`, options: [ - { value: 'gcloud', label: 'Run gcloud Application Default Credentials login' }, { value: 'existing', label: 'Use existing gcloud/Application Default Credentials' }, { value: 'back', label: 'Back' }, ], @@ -535,15 +514,6 @@ async function chooseVertexAuth( if (choice === 'back') { return { status: 'back' }; } - if (choice !== 'gcloud') { - return { status: 'ready' }; - } - - const result = await (deps.runGcloudAuth ?? runKtxSetupGcloudApplicationDefaultAuth)(io); - if (!result.ok) { - io.stderr.write(`gcloud authentication failed: ${result.message}\n`); - return { status: 'missing-input' }; - } return { status: 'ready' }; } @@ -799,7 +769,7 @@ async function chooseVertexModel(args: KtxSetupModelArgs, io: KtxCliIo, deps: Kt return { status: 'missing-input' }; } - const selectableModels = BUNDLED_ANTHROPIC_MODELS.filter(isSelectableAnthropicModel); + const selectableModels = VERTEX_ANTHROPIC_MODELS.filter(isSelectableAnthropicModel); const prompts = deps.prompts ?? createPromptAdapter(); const choice = await prompts.select({ message: `Which Anthropic model should KTX use?\n\n${ANTHROPIC_MODEL_PROMPT_CONTEXT}`, @@ -901,7 +871,7 @@ export async function runKtxSetupAnthropicModelStep( : attemptArgs; if (backendChoice.backend === 'vertex') { - const auth = await chooseVertexAuth(backendArgs, io, deps); + const auth = await chooseVertexAuth(backendArgs, deps); if (auth.status === 'back' && backendChoice.prompted) { attemptArgs = buildInteractiveRetryArgs(args); continue; @@ -931,7 +901,13 @@ export async function runKtxSetupAnthropicModelStep( return { status: model.status, projectDir: args.projectDir }; } - const health = await healthCheck(buildVertexHealthConfig(vertex.values, model.model)); + const health = await runLlmHealthCheckWithProgress( + buildVertexHealthConfig(vertex.values, model.model), + 'Vertex AI', + model.model, + healthCheck, + deps, + ); if (health.ok) { await persistLlmConfig(args.projectDir, { backend: 'vertex', vertex: vertex.refs }, model.model); io.stdout.write(`│ LLM ready: yes (${model.model})\n`); @@ -973,7 +949,13 @@ export async function runKtxSetupAnthropicModelStep( return { status: model.status, projectDir: args.projectDir }; } - const health = await healthCheck(buildAnthropicHealthConfig(credential.value, model.model)); + const health = await runLlmHealthCheckWithProgress( + buildAnthropicHealthConfig(credential.value, model.model), + 'Anthropic API', + model.model, + healthCheck, + deps, + ); if (health.ok) { await persistLlmConfig(args.projectDir, { backend: 'anthropic', credentialRef: credential.ref }, model.model); io.stdout.write(`│ LLM ready: yes (${model.model})\n`); diff --git a/packages/cli/src/setup-sources.test.ts b/packages/cli/src/setup-sources.test.ts index c7dddd0a..2aa7a654 100644 --- a/packages/cli/src/setup-sources.test.ts +++ b/packages/cli/src/setup-sources.test.ts @@ -862,6 +862,7 @@ describe('setup sources step', () => { message: 'Configure dbt', options: [ { value: 'existing:dbt-main', label: 'Use existing dbt connection: dbt-main' }, + { value: 'edit:dbt-main', label: 'Edit existing dbt connection: dbt-main' }, { value: 'new', label: 'Add new dbt connection' }, { value: 'back', label: 'Back' }, ], @@ -989,6 +990,10 @@ describe('setup sources step', () => { value: `existing:${testCase.connectionId}`, label: `Use existing ${testCase.expectedLabel} connection: ${testCase.connectionId}`, }, + { + value: `edit:${testCase.connectionId}`, + label: `Edit existing ${testCase.expectedLabel} connection: ${testCase.connectionId}`, + }, { value: 'new', label: `Add new ${testCase.expectedLabel} connection` }, { value: 'back', label: 'Back' }, ], @@ -997,6 +1002,314 @@ describe('setup sources step', () => { } }); + it('edits an existing Notion source and reopens the page picker with stored pages selected', async () => { + await addPrimarySource(); + await addConnection('notion-main', { + driver: 'notion', + auth_token_ref: 'env:NOTION_TOKEN', + crawl_mode: 'selected_roots', + root_page_ids: ['old-page'], + root_database_ids: [], + root_data_source_ids: [], + }); + const validateNotion = vi.fn(async () => ({ ok: true as const, detail: 'roots=1' })); + const pickNotionRootPages = vi.fn(async () => ({ kind: 'selected' as const, rootPageIds: ['new-page'] })); + const testPrompts = prompts({ + multiselect: [['notion']], + select: ['edit:notion-main', 'keep', 'selected_roots', 'done'], + }); + + await expect( + runKtxSetupSourcesStep( + { projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false }, + makeIo().io, + { + prompts: testPrompts, + validateNotion, + pickNotionRootPages, + }, + ), + ).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['notion-main'] }); + + expect(testPrompts.select).toHaveBeenCalledWith({ + message: 'How should KTX find your Notion integration token?', + options: [ + { value: 'keep', label: 'Keep existing credential' }, + { value: 'env', label: 'Use NOTION_TOKEN from the environment' }, + { value: 'paste', label: 'Paste a key and save it as a local secret file' }, + { value: 'back', label: 'Back' }, + ], + }); + expect(pickNotionRootPages).toHaveBeenCalledWith( + { + connectionId: 'notion-main', + connection: expect.objectContaining({ + driver: 'notion', + auth_token_ref: 'env:NOTION_TOKEN', + crawl_mode: 'selected_roots', + root_page_ids: ['old-page'], + }), + }, + expect.anything(), + ); + expect((await readConfig()).connections['notion-main']).toMatchObject({ + driver: 'notion', + auth_token_ref: 'env:NOTION_TOKEN', + crawl_mode: 'selected_roots', + root_page_ids: ['new-page'], + }); + }); + + it('edits an existing Metabase source with the current URL and credential as defaults', async () => { + await addPrimarySource(); + await addConnection('metabase-main', { + driver: 'metabase', + api_url: 'https://metabase-old.example.com', + api_key_ref: 'env:METABASE_API_KEY', // pragma: allowlist secret + mappings: { + databaseMappings: { '1': 'warehouse' }, + syncEnabled: { '1': true }, + syncMode: 'ALL', + }, + }); + const testPrompts = prompts({ + multiselect: [['metabase']], + select: ['edit:metabase-main', 'keep', 'done'], + text: ['https://metabase-new.example.com'], + }); + const discoverMetabaseDatabases = vi.fn(async () => [ + { id: 2, name: 'Analytics', engine: 'postgres', host: 'db.example.com', dbName: 'analytics' }, + ]); + + await expect( + runKtxSetupSourcesStep( + { projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false }, + makeIo().io, + { + prompts: testPrompts, + discoverMetabaseDatabases, + validateMetabase: vi.fn(async () => ({ ok: true as const, detail: 'mapping validated' })), + runMapping: vi.fn(async () => 0), + }, + ), + ).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['metabase-main'] }); + + expect(testPrompts.text).toHaveBeenCalledWith({ + message: textInputPrompt('Metabase URL'), + initialValue: 'https://metabase-old.example.com', + }); + expect(testPrompts.select).toHaveBeenCalledWith({ + message: 'How should KTX find your Metabase API key?', + options: [ + { value: 'keep', label: 'Keep existing credential' }, + { value: 'env', label: 'Use METABASE_API_KEY from the environment' }, + { value: 'paste', label: 'Paste a key and save it as a local secret file' }, + { value: 'back', label: 'Back' }, + ], + }); + expect(discoverMetabaseDatabases).toHaveBeenCalledWith({ + sourceUrl: 'https://metabase-new.example.com', + sourceApiKeyRef: 'env:METABASE_API_KEY', + sourceConnectionId: 'metabase-main', + }); + expect((await readConfig()).connections['metabase-main']).toMatchObject({ + driver: 'metabase', + api_url: 'https://metabase-new.example.com', + api_key_ref: 'env:METABASE_API_KEY', + mappings: { + databaseMappings: { '2': 'warehouse' }, + syncEnabled: { '2': true }, + syncMode: 'ALL', + }, + }); + }); + + it('rolls back an edited context source when validation fails', async () => { + await addPrimarySource(); + await addConnection('dbt-main', { + driver: 'dbt', + source_dir: '/repo/existing-dbt', + project_name: 'analytics', + }); + const validateDbt = vi.fn(async () => ({ ok: false as const, message: 'dbt project not found' })); + const testPrompts = prompts({ + multiselect: [['dbt']], + select: ['edit:dbt-main', 'path'], + text: ['/repo/new-dbt', ''], + }); + + await expect( + runKtxSetupSourcesStep( + { projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false }, + makeIo().io, + { + prompts: testPrompts, + validateDbt, + }, + ), + ).resolves.toEqual({ status: 'failed', projectDir }); + + expect(validateDbt).toHaveBeenCalledWith(expect.objectContaining({ + driver: 'dbt', + source_dir: '/repo/new-dbt', + })); + const config = await readConfig(); + expect(config.connections['dbt-main']).toMatchObject({ + driver: 'dbt', + source_dir: '/repo/existing-dbt', + project_name: 'analytics', + }); + expect(config.ingest.adapters).not.toContain('dbt'); + }); + + it('lets git-backed context source edits keep the existing repo credential', async () => { + await addPrimarySource(); + await addConnection('metricflow-main', { + driver: 'metricflow', + metricflow: { + repoUrl: 'https://github.com/acme/private-metricflow', + branch: 'main', + path: 'metrics', + auth_token_ref: 'env:METRICFLOW_REPO_TOKEN', // pragma: allowlist secret + }, + }); + const testGitRepo = vi.fn(async () => ({ ok: false as const, error: 'authentication required' })); + const testPrompts = prompts({ + multiselect: [['metricflow']], + select: ['edit:metricflow-main', 'git', 'keep', 'done'], + text: ['https://github.com/acme/private-metricflow', 'main', 'metrics'], + }); + + await expect( + runKtxSetupSourcesStep( + { projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false }, + makeIo().io, + { + prompts: testPrompts, + testGitRepo, + validateMetricflow: vi.fn(async () => ({ ok: true as const, detail: 'metrics=1' })), + }, + ), + ).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['metricflow-main'] }); + + expect(testPrompts.select).toHaveBeenCalledWith({ + message: 'This MetricFlow repo requires authentication.', + options: [ + { value: 'keep', label: 'Keep existing credential' }, + { value: 'env', label: 'Use GITHUB_TOKEN from the environment' }, + { value: 'paste', label: 'Paste a token and save it as a local secret file' }, + { value: 'skip', label: 'Skip — try without authentication' }, + { value: 'back', label: 'Back' }, + ], + }); + expect((await readConfig()).connections['metricflow-main']).toMatchObject({ + driver: 'metricflow', + metricflow: { + repoUrl: 'https://github.com/acme/private-metricflow', + branch: 'main', + path: 'metrics', + auth_token_ref: 'env:METRICFLOW_REPO_TOKEN', + }, + }); + }); + + it('edits an existing context source from the configured-source follow-up menu', async () => { + await addPrimarySource(); + await addConnection('dbt-main', { + driver: 'dbt', + source_dir: '/repo/existing-dbt', + project_name: 'analytics', + }); + const validateDbt = vi.fn(async () => ({ ok: true as const, detail: 'project=analytics schemas=2' })); + const testPrompts = prompts({ + multiselect: [['dbt']], + select: ['existing:dbt-main', 'edit', 'dbt-main', 'path', 'done'], + text: ['/repo/edited-dbt', ''], + }); + + await expect( + runKtxSetupSourcesStep( + { projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false }, + makeIo().io, + { + prompts: testPrompts, + validateDbt, + }, + ), + ).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['dbt-main'] }); + + expect(testPrompts.select).toHaveBeenCalledWith({ + message: '1 context source configured (dbt-main). Add another?', + options: [ + { value: 'done', label: 'Done — continue to context build' }, + { value: 'edit', label: 'Edit an existing context source' }, + { value: 'add', label: 'Add another context source' }, + ], + }); + expect(testPrompts.select).toHaveBeenCalledWith({ + message: 'Context source to edit', + options: [ + { value: 'dbt-main', label: 'dbt-main (dbt)' }, + { value: 'back', label: 'Back' }, + ], + }); + expect(testPrompts.text).toHaveBeenCalledWith({ + message: textInputPrompt('dbt local path'), + initialValue: '/repo/existing-dbt', + }); + expect(validateDbt).toHaveBeenLastCalledWith(expect.objectContaining({ + driver: 'dbt', + source_dir: '/repo/edited-dbt', + })); + expect((await readConfig()).connections['dbt-main']).toMatchObject({ + driver: 'dbt', + source_dir: '/repo/edited-dbt', + project_name: 'analytics', + }); + }); + + it('backs out of editing an existing context source to the source connection menu', async () => { + await addPrimarySource(); + await addConnection('dbt-main', { + driver: 'dbt', + source_dir: '/repo/existing-dbt', + project_name: 'analytics', + }); + const validateDbt = vi.fn(async () => ({ ok: true as const, detail: 'project=analytics schemas=2' })); + const testPrompts = prompts({ + multiselect: [['dbt']], + select: ['edit:dbt-main', 'back', 'existing:dbt-main'], + }); + + await expect( + runKtxSetupSourcesStep( + { projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false }, + makeIo().io, + { + prompts: testPrompts, + validateDbt, + }, + ), + ).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['dbt-main'] }); + + expect( + vi + .mocked(testPrompts.select) + .mock.calls.map(([options]) => options.message) + .filter((message) => message === 'Configure dbt'), + ).toHaveLength(2); + expect(validateDbt).toHaveBeenCalledWith({ + driver: 'dbt', + source_dir: '/repo/existing-dbt', + project_name: 'analytics', + }); + expect((await readConfig()).connections['dbt-main']).toMatchObject({ + driver: 'dbt', + source_dir: '/repo/existing-dbt', + project_name: 'analytics', + }); + }); + it('lets Escape from dbt git URL return to source location selection', async () => { await addPrimarySource(); const validateDbt = vi.fn(async () => ({ ok: true as const, detail: 'project=analytics schemas=2' })); diff --git a/packages/cli/src/setup-sources.ts b/packages/cli/src/setup-sources.ts index c1a923b0..a1ca531a 100644 --- a/packages/cli/src/setup-sources.ts +++ b/packages/cli/src/setup-sources.ts @@ -224,17 +224,20 @@ async function chooseSourceCredentialRef(input: { label: string; envName: string; secretFileName: string; + existingRef?: string; }): Promise { while (true) { const choice = await input.prompts.select({ message: `How should KTX find your ${input.label}?`, options: [ + ...(input.existingRef ? [{ value: 'keep', label: 'Keep existing credential' }] : []), { value: 'env', label: `Use ${input.envName} from the environment` }, { value: 'paste', label: 'Paste a key and save it as a local secret file' }, { value: 'back', label: 'Back' }, ], }); if (choice === 'back') return 'back'; + if (choice === 'keep' && input.existingRef) return input.existingRef; if (choice === 'paste') { const value = await input.prompts.password({ message: input.label }); if (value === undefined) continue; @@ -256,12 +259,14 @@ async function chooseGitAuthCredentialRef(input: { projectDir: string; source: KtxSetupSourceType; connectionId: string; + existingRef?: string; }): Promise { const label = input.source === 'dbt' ? 'This' : `This ${sourceLabel(input.source)}`; while (true) { const choice = await input.prompts.select({ message: `${label} repo requires authentication.`, options: [ + ...(input.existingRef ? [{ value: 'keep', label: 'Keep existing credential' }] : []), { value: 'env', label: 'Use GITHUB_TOKEN from the environment' }, { value: 'paste', label: 'Paste a token and save it as a local secret file' }, { value: 'skip', label: 'Skip — try without authentication' }, @@ -269,6 +274,7 @@ async function chooseGitAuthCredentialRef(input: { ], }); if (choice === 'back') return 'back'; + if (choice === 'keep' && input.existingRef) return input.existingRef; if (choice === 'skip') return undefined; if (choice === 'paste') { const value = await input.prompts.password({ message: 'Git access token' }); @@ -792,8 +798,14 @@ interface WarehouseConnectionChoice { type InteractiveSourceConnectionChoice = | { kind: 'existing'; connectionId: string; connection: KtxProjectConnectionConfig } | { kind: 'new'; args: KtxSetupSourcesArgs } + | { kind: 'edited'; connectionId: string; args: KtxSetupSourcesArgs } | 'back'; +type SourceSetupChoiceResult = + | { status: 'ready'; connectionId: string } + | { status: 'back' } + | { status: 'failed' }; + async function runSourcePromptSteps( initialState: SourcePromptState, stepsForState: (state: SourcePromptState) => SourcePromptStep[], @@ -827,6 +839,12 @@ function resetRepoLocationFields(state: SourcePromptState): void { delete state.sourceProjectName; } +function sourceLocationFromArgs(args: KtxSetupSourcesArgs): SourceLocationChoice | undefined { + if (args.sourcePath) return 'path'; + if (args.sourceGitUrl) return 'git'; + return undefined; +} + function warehouseConnectionChoices(config: KtxProjectConfig): WarehouseConnectionChoice[] { return Object.entries(config.connections) .filter(([, connection]) => PRIMARY_SOURCE_DRIVERS.has(String(connection.driver ?? '').toLowerCase())) @@ -963,7 +981,7 @@ async function promptForInteractiveSource( testGitRepo: KtxSetupSourcesDeps['testGitRepo'] = testRepoConnection, discoverMetabaseDatabaseList?: KtxSetupSourcesDeps['discoverMetabaseDatabases'], ): Promise { - const initialState: SourcePromptState = { ...args, source }; + const initialState: SourcePromptState = { ...args, source, sourceLocation: sourceLocationFromArgs(args) }; if (args.sourceConnectionId) { initialState.sourceConnectionId = args.sourceConnectionId; } @@ -993,7 +1011,10 @@ async function promptForInteractiveSource( ...(state.sourceLocation === 'path' ? [ async (currentState: SourcePromptState) => { - const sourcePath = await promptText(prompts, { message: `${source} local path` }); + const sourcePath = await promptText(prompts, { + message: `${source} local path`, + ...(currentState.sourcePath ? { initialValue: currentState.sourcePath } : {}), + }); if (sourcePath === undefined) return 'back'; currentState.sourcePath = sourcePath; return 'next'; @@ -1003,13 +1024,19 @@ async function promptForInteractiveSource( ...(state.sourceLocation === 'git' ? [ async (currentState: SourcePromptState) => { - const sourceGitUrl = await promptText(prompts, { message: `${source} git URL` }); + const sourceGitUrl = await promptText(prompts, { + message: `${source} git URL`, + ...(currentState.sourceGitUrl ? { initialValue: currentState.sourceGitUrl } : {}), + }); if (sourceGitUrl === undefined) return 'back'; currentState.sourceGitUrl = sourceGitUrl; return 'next'; }, async (currentState: SourcePromptState) => { - const branch = await promptText(prompts, { message: `${source} git branch`, initialValue: 'main' }); + const branch = await promptText(prompts, { + message: `${source} git branch`, + initialValue: currentState.sourceBranch ?? 'main', + }); if (branch === undefined) return 'back'; currentState.sourceBranch = branch || 'main'; return 'next'; @@ -1030,6 +1057,7 @@ async function promptForInteractiveSource( projectDir: args.projectDir, source, connectionId: currentState.sourceConnectionId ?? `${source}-main`, + existingRef: currentState.sourceAuthTokenRef, }); if (authRef === 'back') return 'back'; if (authRef) { @@ -1103,6 +1131,7 @@ async function promptForInteractiveSource( const subpath = await promptText(prompts, { message: sourceSubpathPrompt(source), placeholder: 'optional', + ...(currentState.sourceSubpath ? { initialValue: currentState.sourceSubpath } : {}), }); if (subpath === undefined) return 'back'; if (subpath) { @@ -1121,7 +1150,10 @@ async function promptForInteractiveSource( return await runSourcePromptSteps(initialState, () => [ ...connectionSteps, async (state) => { - const sourceUrl = await promptText(prompts, { message: 'Metabase URL' }); + const sourceUrl = await promptText(prompts, { + message: 'Metabase URL', + ...(state.sourceUrl ? { initialValue: state.sourceUrl } : {}), + }); if (sourceUrl === undefined) return 'back'; state.sourceUrl = sourceUrl; return 'next'; @@ -1133,6 +1165,7 @@ async function promptForInteractiveSource( label: 'Metabase API key', envName: 'METABASE_API_KEY', secretFileName: `${state.sourceConnectionId ?? 'metabase-main'}-api-key`, + existingRef: state.sourceApiKeyRef, }); if (ref === 'back') return 'back'; state.sourceApiKeyRef = ref; @@ -1164,13 +1197,19 @@ async function promptForInteractiveSource( return await runSourcePromptSteps(initialState, () => [ ...connectionSteps, async (state) => { - const sourceUrl = await promptText(prompts, { message: 'Looker base URL' }); + const sourceUrl = await promptText(prompts, { + message: 'Looker base URL', + ...(state.sourceUrl ? { initialValue: state.sourceUrl } : {}), + }); if (sourceUrl === undefined) return 'back'; state.sourceUrl = sourceUrl; return 'next'; }, async (state) => { - const sourceClientId = await promptText(prompts, { message: 'Looker client id' }); + const sourceClientId = await promptText(prompts, { + message: 'Looker client id', + ...(state.sourceClientId ? { initialValue: state.sourceClientId } : {}), + }); if (sourceClientId === undefined) return 'back'; state.sourceClientId = sourceClientId; return 'next'; @@ -1182,6 +1221,7 @@ async function promptForInteractiveSource( label: 'Looker client secret', envName: 'LOOKER_CLIENT_SECRET', secretFileName: `${state.sourceConnectionId ?? 'looker-main'}-client-secret`, + existingRef: state.sourceClientSecretRef, }); if (ref === 'back') return 'back'; state.sourceClientSecretRef = ref; @@ -1200,6 +1240,7 @@ async function promptForInteractiveSource( const lookerConnectionName = await promptText(prompts, { message: 'Looker connection name', placeholder: 'optional', + ...(state.sourceTarget ? { initialValue: state.sourceTarget } : {}), }); if (lookerConnectionName === undefined) return 'back'; if (lookerConnectionName) { @@ -1221,6 +1262,7 @@ async function promptForInteractiveSource( label: 'Notion integration token', envName: 'NOTION_TOKEN', secretFileName: `${currentState.sourceConnectionId ?? 'notion-main'}-token`, + existingRef: currentState.sourceApiKeyRef, }); if (ref === 'back') return 'back'; currentState.sourceApiKeyRef = ref; @@ -1285,6 +1327,24 @@ function existingConnectionIdsBySource( .sort((left, right) => left.localeCompare(right)); } +function sourceTypeForConnection(connection: KtxProjectConnectionConfig): KtxSetupSourceType | null { + const driver = String(connection.driver ?? '').toLowerCase(); + return SOURCE_OPTIONS.some((option) => option.value === driver) ? (driver as KtxSetupSourceType) : null; +} + +function contextSourceEditTargets(connections: Record): Array<{ + connectionId: string; + source: KtxSetupSourceType; +}> { + return Object.entries(connections) + .map(([connectionId, connection]) => { + const source = sourceTypeForConnection(connection); + return source ? { connectionId, source } : null; + }) + .filter((target): target is { connectionId: string; source: KtxSetupSourceType } => target !== null) + .sort((left, right) => left.connectionId.localeCompare(right.connectionId)); +} + function sourceChecklistForConnections(connections: Record): { options: Array<{ value: KtxSetupSourceType; label: string; hint?: string }>; initialValues: KtxSetupSourceType[]; @@ -1316,6 +1376,180 @@ function defaultConnectionIdForSource( return `${base}-${index}`; } +function firstStringRecordEntry(value: unknown): [string, string] | undefined { + if (!isRecord(value)) return undefined; + for (const [key, raw] of Object.entries(value)) { + if (typeof raw === 'string' && raw.trim().length > 0) { + return [key, raw.trim()]; + } + } + return undefined; +} + +function applyRepoSourceArgs( + args: KtxSetupSourcesArgs, + input: { repoUrl?: string; sourceDir?: string; branch?: string; subpath?: string; authTokenRef?: string }, +): void { + if (input.sourceDir) { + args.sourcePath = input.sourceDir; + } else if (input.repoUrl?.startsWith('file:')) { + args.sourcePath = fileURLToPath(input.repoUrl); + } else if (input.repoUrl) { + args.sourceGitUrl = input.repoUrl; + } + if (input.branch) args.sourceBranch = input.branch; + if (input.subpath) args.sourceSubpath = input.subpath; + if (input.authTokenRef) args.sourceAuthTokenRef = input.authTokenRef; +} + +function sourceArgsFromExistingConnection(input: { + args: KtxSetupSourcesArgs; + source: KtxSetupSourceType; + connectionId: string; + connection: KtxProjectConnectionConfig; +}): KtxSetupSourcesArgs { + const sourceArgs: KtxSetupSourcesArgs = { + projectDir: input.args.projectDir, + inputMode: input.args.inputMode, + source: input.source, + sourceConnectionId: input.connectionId, + runInitialSourceIngest: input.args.runInitialSourceIngest, + skipSources: input.args.skipSources, + }; + + if (input.source === 'dbt') { + applyRepoSourceArgs(sourceArgs, { + sourceDir: stringField(input.connection.source_dir), + repoUrl: stringField(input.connection.repo_url), + branch: stringField(input.connection.branch), + subpath: stringField(input.connection.path), + authTokenRef: stringField(input.connection.auth_token_ref), + }); + const profilesPath = stringField(input.connection.profiles_path); + const target = stringField(input.connection.target); + const projectName = stringField(input.connection.project_name); + if (profilesPath) sourceArgs.sourceProfilesPath = profilesPath; + if (target) sourceArgs.sourceTarget = target; + if (projectName) sourceArgs.sourceProjectName = projectName; + return sourceArgs; + } + + if (input.source === 'metricflow') { + const metricflow = isRecord(input.connection.metricflow) ? input.connection.metricflow : {}; + applyRepoSourceArgs(sourceArgs, { + repoUrl: stringField(metricflow.repoUrl), + branch: stringField(metricflow.branch), + subpath: stringField(metricflow.path), + authTokenRef: stringField(metricflow.auth_token_ref), + }); + return sourceArgs; + } + + if (input.source === 'lookml') { + applyRepoSourceArgs(sourceArgs, { + repoUrl: stringField(input.connection.repoUrl), + branch: stringField(input.connection.branch), + subpath: stringField(input.connection.path), + authTokenRef: stringField(input.connection.auth_token_ref), + }); + const mappings = isRecord(input.connection.mappings) ? input.connection.mappings : {}; + const expectedLookerConnectionName = stringField(mappings.expectedLookerConnectionName); + if (expectedLookerConnectionName) sourceArgs.sourceTarget = expectedLookerConnectionName; + return sourceArgs; + } + + if (input.source === 'metabase') { + sourceArgs.sourceUrl = stringField(input.connection.api_url); + sourceArgs.sourceApiKeyRef = stringField(input.connection.api_key_ref); + const mappings = isRecord(input.connection.mappings) ? input.connection.mappings : {}; + const databaseMapping = firstStringRecordEntry(mappings.databaseMappings); + if (databaseMapping) { + sourceArgs.metabaseDatabaseId = Number.parseInt(databaseMapping[0], 10); + sourceArgs.sourceWarehouseConnectionId = databaseMapping[1]; + } + return sourceArgs; + } + + if (input.source === 'looker') { + sourceArgs.sourceUrl = stringField(input.connection.base_url); + sourceArgs.sourceClientId = stringField(input.connection.client_id); + sourceArgs.sourceClientSecretRef = stringField(input.connection.client_secret_ref); + const mappings = isRecord(input.connection.mappings) ? input.connection.mappings : {}; + const connectionMapping = firstStringRecordEntry(mappings.connectionMappings); + if (connectionMapping) { + sourceArgs.sourceTarget = connectionMapping[0]; + sourceArgs.sourceWarehouseConnectionId = connectionMapping[1]; + } + return sourceArgs; + } + + sourceArgs.sourceApiKeyRef = stringField(input.connection.auth_token_ref); + sourceArgs.notionCrawlMode = + input.connection.crawl_mode === 'all_accessible' ? 'all_accessible' : 'selected_roots'; + if (Array.isArray(input.connection.root_page_ids)) { + sourceArgs.notionRootPageIds = input.connection.root_page_ids.filter( + (pageId): pageId is string => typeof pageId === 'string', + ); + } + return sourceArgs; +} + +async function promptEditedSourceConnection(input: { + args: KtxSetupSourcesArgs; + source: KtxSetupSourceType; + connectionId: string; + connection: KtxProjectConnectionConfig; + prompts: KtxSetupSourcesPromptAdapter; + io: KtxCliIo; + testGitRepo?: KtxSetupSourcesDeps['testGitRepo']; + pickNotionRootPages?: KtxSetupSourcesDeps['pickNotionRootPages']; + discoverMetabaseDatabases?: KtxSetupSourcesDeps['discoverMetabaseDatabases']; +}): Promise | 'back'> { + const sourceArgs = await promptForInteractiveSource( + sourceArgsFromExistingConnection({ + args: input.args, + source: input.source, + connectionId: input.connectionId, + connection: input.connection, + }), + input.source, + input.prompts, + input.io, + { + pickNotionRootPages: input.pickNotionRootPages, + discoverMetabaseDatabases: input.discoverMetabaseDatabases, + }, + input.connectionId, + input.testGitRepo, + input.discoverMetabaseDatabases, + ); + return sourceArgs === 'back' + ? 'back' + : { kind: 'edited', connectionId: input.connectionId, args: sourceArgs }; +} + +async function chooseContextSourceToEdit(input: { + projectDir: string; + prompts: KtxSetupSourcesPromptAdapter; +}): Promise<{ connectionId: string; source: KtxSetupSourceType } | 'back'> { + const project = await loadKtxProject({ projectDir: input.projectDir }); + const targets = contextSourceEditTargets(project.config.connections); + if (targets.length === 0) return 'back'; + const choice = await input.prompts.select({ + message: 'Context source to edit', + options: [ + ...targets.map((target) => ({ + value: target.connectionId, + label: `${target.connectionId} (${sourceLabel(target.source)})`, + })), + { value: 'back', label: 'Back' }, + ], + }); + if (choice === 'back') return 'back'; + const target = targets.find((candidate) => candidate.connectionId === choice); + return target ?? 'back'; +} + async function chooseInteractiveSourceConnection(input: { args: KtxSetupSourcesArgs; source: KtxSetupSourceType; @@ -1355,6 +1589,10 @@ async function chooseInteractiveSourceConnection(input: { value: `existing:${connectionId}`, label: `Use existing ${label} connection: ${connectionId}`, })), + ...existingIds.map((connectionId) => ({ + value: `edit:${connectionId}`, + label: `Edit existing ${label} connection: ${connectionId}`, + })), { value: 'new', label: `Add new ${label} connection` }, { value: 'back', label: 'Back' }, ], @@ -1368,6 +1606,28 @@ async function chooseInteractiveSourceConnection(input: { } continue; } + if (choice.startsWith('edit:')) { + const connectionId = choice.slice('edit:'.length); + const connection = input.connections[connectionId]; + if (!connection) { + continue; + } + const edited = await promptEditedSourceConnection({ + args: input.args, + source: input.source, + connectionId, + connection, + prompts: input.prompts, + io: input.io, + testGitRepo: input.testGitRepo, + pickNotionRootPages: input.pickNotionRootPages, + discoverMetabaseDatabases: input.discoverMetabaseDatabases, + }); + if (edited === 'back') { + continue; + } + return edited; + } const sourceArgs = await promptForInteractiveSource( input.args, input.source, @@ -1432,6 +1692,85 @@ async function validateSource( return await (deps.validateNotion ?? defaultValidateNotion)(args.connection); } +async function saveValidateAndMaybeBuildSource(input: { + args: KtxSetupSourcesArgs; + source: KtxSetupSourceType; + sourceChoice: Exclude; + prompts: KtxSetupSourcesPromptAdapter; + io: KtxCliIo; + deps: KtxSetupSourcesDeps; +}): Promise { + const connectionId = + input.sourceChoice.kind === 'existing' + ? input.sourceChoice.connectionId + : input.sourceChoice.kind === 'edited' + ? input.sourceChoice.connectionId + : (input.sourceChoice.args.sourceConnectionId ?? `${input.source}-main`); + const connection = + input.sourceChoice.kind === 'existing' + ? input.sourceChoice.connection + : buildConnection(input.source, input.sourceChoice.args); + const rollback = + input.sourceChoice.kind === 'existing' + ? undefined + : await writeSourceConnection( + input.args.projectDir, + connectionId, + connection, + sourceAdapter(input.source), + ); + + if (input.sourceChoice.kind === 'existing') { + await ensureSourceAdapterEnabled(input.args.projectDir, input.source); + } + + const validation = await validateSource( + input.source, + { projectDir: input.args.projectDir, connectionId, connection }, + input.deps, + ); + if (!validation.ok) { + await rollback?.(); + input.io.stderr.write(`${validation.message}\n`); + return { status: 'failed' }; + } + + if (input.source === 'metabase' || input.source === 'looker') { + input.prompts.log?.(`Validating ${sourceLabel(input.source)} mapping…`); + const mappingCode = await (input.deps.runMapping ?? defaultRunMapping)( + input.args.projectDir, + connectionId, + createSetupPrefixedIo(input.io), + ); + if (mappingCode !== 0) { + await rollback?.(); + return { status: 'failed' }; + } + } + + if (input.args.runInitialSourceIngest) { + const ingestResult = await runInitialSourceIngestWithRecovery({ + args: input.args, + connectionId, + io: input.io, + prompts: input.prompts, + deps: input.deps, + }); + if (ingestResult === 'failed') { + await rollback?.(); + return { status: 'failed' }; + } + if (ingestResult === 'back') { + await rollback?.(); + return { status: 'back' }; + } + } else { + input.io.stdout.write(`│ Context source ${connectionId} saved. It will be built during the context build step.\n`); + } + + return { status: 'ready', connectionId }; +} + export async function runKtxSetupSourcesStep( args: KtxSetupSourcesArgs, io: KtxCliIo, @@ -1509,62 +1848,27 @@ export async function runKtxSetupSourcesStep( returnToSourceSelection = true; break; } - const connectionId = - sourceChoice.kind === 'existing' - ? sourceChoice.connectionId - : (sourceChoice.args.sourceConnectionId ?? `${source}-main`); - const connection = - sourceChoice.kind === 'existing' ? sourceChoice.connection : buildConnection(source, sourceChoice.args); - const rollback = - sourceChoice.kind === 'existing' - ? undefined - : await writeSourceConnection(args.projectDir, connectionId, connection, sourceAdapter(source)); - if (sourceChoice.kind === 'existing') { - await ensureSourceAdapterEnabled(args.projectDir, source); - } - const validation = await validateSource(source, { projectDir: args.projectDir, connectionId, connection }, deps); - - if (!validation.ok) { - await rollback?.(); - io.stderr.write(`${validation.message}\n`); + const choiceResult = await saveValidateAndMaybeBuildSource({ + args, + source, + sourceChoice, + prompts, + io, + deps, + }); + if (choiceResult.status === 'failed') { return { status: 'failed', projectDir: args.projectDir }; } - if (source === 'metabase' || source === 'looker') { - prompts.log?.(`Validating ${sourceLabel(source)} mapping…`); - const mappingCode = await (deps.runMapping ?? defaultRunMapping)( - args.projectDir, - connectionId, - createSetupPrefixedIo(io), - ); - if (mappingCode !== 0) { - await rollback?.(); - return { status: 'failed', projectDir: args.projectDir }; + if (choiceResult.status === 'back') { + if (args.source) { + return { status: 'back', projectDir: args.projectDir }; } + returnToSourceSelection = true; + break; } - if (args.runInitialSourceIngest) { - const ingestResult = await runInitialSourceIngestWithRecovery({ - args, - connectionId, - io, - prompts, - deps, - }); - if (ingestResult === 'failed') { - await rollback?.(); - return { status: 'failed', projectDir: args.projectDir }; - } - if (ingestResult === 'back') { - await rollback?.(); - if (args.source) { - return { status: 'back', projectDir: args.projectDir }; - } - returnToSourceSelection = true; - break; - } - } else { - io.stdout.write(`│ Context source ${connectionId} saved. It will be built during the context build step.\n`); + if (!readyConnectionIds.includes(choiceResult.connectionId)) { + readyConnectionIds.push(choiceResult.connectionId); } - readyConnectionIds.push(connectionId); } if (returnToSourceSelection) { @@ -1572,14 +1876,66 @@ export async function runKtxSetupSourcesStep( } if (readyConnectionIds.length > 0 && !args.source && args.inputMode !== 'disabled') { - const addMore = await prompts.select({ - message: `${readyConnectionIds.length} context source${readyConnectionIds.length > 1 ? 's' : ''} configured (${readyConnectionIds.join(', ')}). Add another?`, - options: [ - { value: 'done', label: 'Done — continue to context build' }, - { value: 'add', label: 'Add another context source' }, - ], - }); - if (addMore === 'add') { + let restartSourceSelection = false; + while (true) { + const addMore = await prompts.select({ + message: `${readyConnectionIds.length} context source${readyConnectionIds.length > 1 ? 's' : ''} configured (${readyConnectionIds.join(', ')}). Add another?`, + options: [ + { value: 'done', label: 'Done — continue to context build' }, + { value: 'edit', label: 'Edit an existing context source' }, + { value: 'add', label: 'Add another context source' }, + ], + }); + if (addMore === 'add') { + restartSourceSelection = true; + break; + } + if (addMore === 'edit') { + const editTarget = await chooseContextSourceToEdit({ projectDir: args.projectDir, prompts }); + if (editTarget === 'back') { + continue; + } + const projectForEdit = await loadKtxProject({ projectDir: args.projectDir }); + const connection = projectForEdit.config.connections[editTarget.connectionId]; + if (!connection) { + continue; + } + const sourceChoice = await promptEditedSourceConnection({ + args, + source: editTarget.source, + connectionId: editTarget.connectionId, + connection, + prompts, + io, + testGitRepo: deps.testGitRepo, + pickNotionRootPages: deps.pickNotionRootPages, + discoverMetabaseDatabases: deps.discoverMetabaseDatabases, + }); + if (sourceChoice === 'back') { + continue; + } + const choiceResult = await saveValidateAndMaybeBuildSource({ + args, + source: editTarget.source, + sourceChoice, + prompts, + io, + deps, + }); + if (choiceResult.status === 'failed') { + return { status: 'failed', projectDir: args.projectDir }; + } + if (choiceResult.status === 'back') { + continue; + } + if (!readyConnectionIds.includes(choiceResult.connectionId)) { + readyConnectionIds.push(choiceResult.connectionId); + } + continue; + } + break; + } + if (restartSourceSelection) { continue; } } diff --git a/packages/cli/src/status-project.ts b/packages/cli/src/status-project.ts new file mode 100644 index 00000000..59afa811 --- /dev/null +++ b/packages/cli/src/status-project.ts @@ -0,0 +1,667 @@ +import type { + KtxLocalProject, + KtxProjectConfig, + KtxProjectConnectionConfig, + KtxProjectEmbeddingConfig, + KtxProjectLlmConfig, +} from '@ktx/context/project'; +import type { DoctorCheck } from './doctor.js'; + +type ProjectStatusLevel = 'ok' | 'warn' | 'fail'; +type ProjectVerdict = 'ready' | 'partial' | 'blocked'; + +interface ProjectStatusLine { + status: ProjectStatusLevel; + detail: string; + fix?: string; +} + +interface LlmStatus extends ProjectStatusLine { + backend: string; + model?: string; +} + +interface EmbeddingsStatus extends ProjectStatusLine { + backend: string; + model?: string; + dimensions?: number; +} + +interface ConnectionStatus extends ProjectStatusLine { + name: string; + driver: string; +} + +interface PipelineStatus { + adapters: string[]; + enrichmentMode: string; + relationshipsEnabled: boolean; + relationshipsLlmProposals: boolean; + relationshipsValidationRequired: boolean; + agentEnabled: boolean; + agentTools: string[]; + agentMaxIterations: number; +} + +interface StorageStatus { + state: string; + search: string; + gitAutoCommit: boolean; + gitAuthor: string; +} + +interface WarningItem { + message: string; + fix?: string; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function hasOwnField(value: Record, key: string): boolean { + return Object.prototype.hasOwnProperty.call(value, key); +} + +export interface ProjectStatus { + projectName: string; + projectDir: string; + llm: LlmStatus; + embeddings: EmbeddingsStatus; + storage: StorageStatus; + connections: ConnectionStatus[]; + pipeline: PipelineStatus; + warnings: WarningItem[]; + verdict: ProjectVerdict; + verdictReason: string; + nextActions: string[]; + promptCaching?: { enabled: boolean; systemTtl?: string; toolsTtl?: string; historyTtl?: string }; + workUnits?: { stepBudget: number; maxConcurrency: number; failureMode: string }; + memoryAutoCommit: boolean; + relationshipsDetail?: { + acceptThreshold: number; + reviewThreshold: number; + maxLlmTablesPerBatch: number; + validationConcurrency: number; + }; +} + +function resolveRef(value: unknown, env: NodeJS.ProcessEnv): { resolved: string; via: 'literal' | 'env' | 'file' | 'missing' } { + if (typeof value !== 'string') return { resolved: '', via: 'missing' }; + const trimmed = value.trim(); + if (trimmed.length === 0) return { resolved: '', via: 'missing' }; + if (trimmed.startsWith('env:')) { + const name = trimmed.slice(4).trim(); + const v = env[name]; + return v && v.trim().length > 0 ? { resolved: v, via: 'env' } : { resolved: '', via: 'missing' }; + } + if (trimmed.startsWith('file:')) { + return { resolved: trimmed.slice(5), via: 'file' }; + } + return { resolved: trimmed, via: 'literal' }; +} + +function envHint(value: unknown): string | undefined { + if (typeof value === 'string' && value.trim().startsWith('env:')) { + return value.trim().slice(4).trim(); + } + return undefined; +} + +function buildLlmStatus(config: KtxProjectLlmConfig, env: NodeJS.ProcessEnv): LlmStatus { + const backend = config.provider.backend; + const model = config.models?.default; + if (backend === 'none') { + return { + backend, + model, + status: 'fail', + detail: 'no LLM configured — ktx ask will not work', + fix: 'Run: ktx setup (choose an LLM provider)', + }; + } + if (backend === 'anthropic') { + const ref = config.provider.anthropic?.api_key; + const resolved = resolveRef(ref, env); + if (resolved.resolved.length > 0) { + return { backend, model, status: 'ok', detail: `key set${resolved.via === 'env' ? ` (env)` : ''}` }; + } + if (env.ANTHROPIC_API_KEY && env.ANTHROPIC_API_KEY.trim().length > 0) { + return { backend, model, status: 'ok', detail: 'key set (env: ANTHROPIC_API_KEY)' }; + } + const hint = envHint(ref); + return { + backend, + model, + status: 'warn', + detail: hint ? `key missing (env: ${hint})` : 'key missing', + fix: hint ? `Set ${hint}` : 'Set ANTHROPIC_API_KEY or rerun `ktx setup`', + }; + } + if (backend === 'vertex') { + const project = config.provider.vertex?.project; + if (project && project.length > 0) { + return { backend, model, status: 'ok', detail: `project=${project}` }; + } + return { backend, model, status: 'warn', detail: 'vertex project not configured', fix: 'Rerun `ktx setup`' }; + } + if (backend === 'gateway') { + const ref = config.provider.gateway?.api_key; + const resolved = resolveRef(ref, env); + if (resolved.resolved.length > 0) { + return { backend, model, status: 'ok', detail: 'key set' }; + } + const hint = envHint(ref); + return { + backend, + model, + status: 'warn', + detail: hint ? `key missing (env: ${hint})` : 'key missing', + fix: hint ? `Set ${hint}` : 'Set the gateway api_key or rerun `ktx setup`', + }; + } + return { backend, model, status: 'warn', detail: 'unknown LLM backend' }; +} + +function buildEmbeddingsStatus(config: KtxProjectEmbeddingConfig, env: NodeJS.ProcessEnv): EmbeddingsStatus { + const backend = config.backend; + const model = config.model; + const dimensions = config.dimensions; + if (backend === 'none') { + return { + backend, + model, + dimensions, + status: 'warn', + detail: 'disabled — semantic search will be skipped', + }; + } + if (backend === 'deterministic') { + return { + backend, + model, + dimensions, + status: 'warn', + detail: 'deterministic — semantic search degraded (lexical/dictionary lanes still work)', + }; + } + if (backend === 'openai') { + const ref = config.openai?.api_key; + const resolved = resolveRef(ref, env); + if (resolved.resolved.length > 0 || (env.OPENAI_API_KEY && env.OPENAI_API_KEY.trim().length > 0)) { + return { backend, model, dimensions, status: 'ok', detail: 'key set' }; + } + const hint = envHint(ref); + return { + backend, + model, + dimensions, + status: 'warn', + detail: hint ? `key missing (env: ${hint})` : 'key missing', + fix: hint ? `Set ${hint}` : 'Set OPENAI_API_KEY or rerun `ktx setup`', + }; + } + if (backend === 'sentence-transformers') { + const url = config.sentenceTransformers?.base_url; + if (typeof url === 'string' && url.length > 0) { + return { backend, model, dimensions, status: 'ok', detail: `service: ${url}` }; + } + return { + backend, + model, + dimensions, + status: 'warn', + detail: 'no base_url configured', + fix: 'Rerun `ktx setup`', + }; + } + return { backend, model, dimensions, status: 'warn', detail: 'unknown embedding backend' }; +} + +function buildConnectionStatus( + name: string, + conn: KtxProjectConnectionConfig, + env: NodeJS.ProcessEnv, +): ConnectionStatus { + const driver = (conn.driver ?? 'unknown').toLowerCase(); + const ok = (detail: string): ConnectionStatus => ({ name, driver, status: 'ok', detail }); + const warn = (detail: string, fix?: string): ConnectionStatus => ({ name, driver, status: 'warn', detail, fix }); + + switch (driver) { + case 'postgres': + case 'postgresql': + case 'mysql': + case 'clickhouse': + case 'sqlserver': { + const urlRef = resolveRef(conn.url, env); + if (urlRef.resolved.length > 0) return ok(`url configured`); + if (typeof (conn as Record).host === 'string') return ok('host configured'); + const hint = envHint(conn.url); + return warn(hint ? `url missing (env: ${hint})` : 'url not set', hint ? `Set ${hint}` : 'Rerun `ktx setup`'); + } + case 'snowflake': { + const account = (conn as Record).account; + if (typeof account === 'string' && account.length > 0) return ok(`account: ${account}`); + return warn('account not set', 'Rerun `ktx setup`'); + } + case 'bigquery': { + const cred = resolveRef((conn as Record).credentials_json, env); + if (cred.resolved.length > 0) return ok('credentials configured'); + const hint = envHint((conn as Record).credentials_json); + return warn(hint ? `credentials missing (env: ${hint})` : 'credentials not set', hint ? `Set ${hint}` : 'Rerun `ktx setup`'); + } + case 'sqlite': { + const path = (conn as Record).path; + if (typeof path === 'string' && path.length > 0) return ok(`path: ${path}`); + return warn('path not set', 'Rerun `ktx setup`'); + } + case 'notion': { + const tokenRef = + (conn as Record).auth_token_ref ?? + (conn as Record).auth_token; + const resolved = resolveRef(tokenRef, env); + if (resolved.resolved.length > 0) return ok('auth token configured'); + const hint = envHint(tokenRef); + return warn(hint ? `auth token missing (env: ${hint})` : 'auth token not set', hint ? `Set ${hint}` : 'Rerun `ktx setup`'); + } + case 'dbt': + case 'dbt-core': + case 'dbt-cloud': { + const repoUrl = + (conn as Record).repoUrl ?? + (conn as Record).repo_url; + if (typeof repoUrl === 'string' && repoUrl.length > 0) return ok(`repo: ${repoUrl}`); + return warn('repoUrl not set', 'Rerun `ktx setup`'); + } + case 'metabase': { + const url = (conn as Record).url ?? (conn as Record).base_url; + if (typeof url === 'string' && url.length > 0) return ok(`url: ${url}`); + return warn('url not set', 'Rerun `ktx setup`'); + } + case 'looker': + case 'lookml': { + const url = (conn as Record).base_url ?? (conn as Record).url; + if (typeof url === 'string' && url.length > 0) return ok(`url: ${url}`); + return warn('base_url not set', 'Rerun `ktx setup`'); + } + case 'metricflow': { + const repoUrl = (conn as Record).repoUrl ?? (conn as Record).repo_url; + if (typeof repoUrl === 'string' && repoUrl.length > 0) return ok(`repo: ${repoUrl}`); + return warn('repoUrl not set', 'Rerun `ktx setup`'); + } + default: + return { name, driver, status: 'ok', detail: 'configured' }; + } +} + +const ADAPTER_DRIVER_REQUIREMENT: Record = { + 'live-database': ['postgres', 'postgresql', 'mysql', 'snowflake', 'bigquery', 'clickhouse', 'sqlite', 'sqlserver'], + dbt: ['dbt', 'dbt-core', 'dbt-cloud'], + notion: ['notion'], + metabase: ['metabase'], + looker: ['looker', 'lookml'], + lookml: ['looker', 'lookml'], + metricflow: ['metricflow'], +}; + +function buildPipelineStatus(config: KtxProjectConfig): PipelineStatus { + return { + adapters: config.ingest.adapters, + enrichmentMode: config.scan.enrichment.mode, + relationshipsEnabled: config.scan.relationships.enabled, + relationshipsLlmProposals: config.scan.relationships.llmProposals, + relationshipsValidationRequired: config.scan.relationships.validationRequiredForManifest, + agentEnabled: config.agent.run_research.enabled, + agentTools: config.agent.run_research.default_toolset, + agentMaxIterations: config.agent.run_research.max_iterations, + }; +} + +function buildStorageStatus(config: KtxProjectConfig): StorageStatus { + return { + state: config.storage.state, + search: config.storage.search, + gitAutoCommit: config.storage.git.auto_commit, + gitAuthor: config.storage.git.author, + }; +} + +function buildWarnings( + config: KtxProjectConfig, + connections: ConnectionStatus[], + llm: LlmStatus, + embeddings: EmbeddingsStatus, +): WarningItem[] { + const warnings: WarningItem[] = []; + + for (const [connectionId, connection] of Object.entries(config.connections)) { + const driver = String(connection.driver ?? '').toLowerCase(); + if (hasOwnField(connection, 'readonly')) { + warnings.push({ + message: `connections.${connectionId}.readonly is no longer used.`, + fix: `Remove connections.${connectionId}.readonly from ktx.yaml.`, + }); + } + + if ((driver === 'sqlite' || driver === 'sqlite3') && hasOwnField(connection, 'file_path')) { + warnings.push({ + message: `connections.${connectionId}.file_path was removed.`, + fix: `Rename connections.${connectionId}.file_path to path.`, + }); + } + + if (driver === 'notion' && hasOwnField(connection, 'last_successful_cursor')) { + warnings.push({ + message: `connections.${connectionId}.last_successful_cursor is local sync state.`, + fix: 'Remove it from ktx.yaml. KTX stores the Notion cursor in .ktx/db.sqlite.', + }); + } + + const historicSql = isRecord(connection.historicSql) ? connection.historicSql : null; + if (!historicSql) { + continue; + } + if (hasOwnField(historicSql, 'concurrency')) { + warnings.push({ + message: `connections.${connectionId}.historicSql.concurrency is no longer used.`, + fix: `Remove connections.${connectionId}.historicSql.concurrency from ktx.yaml.`, + }); + } + const historicDialect = String(historicSql.dialect ?? driver).toLowerCase(); + if ( + (historicDialect === 'postgres' || historicDialect === 'postgresql') && + hasOwnField(historicSql, 'windowDays') + ) { + warnings.push({ + message: `connections.${connectionId}.historicSql.windowDays does not constrain pg_stat_statements.`, + fix: `Remove connections.${connectionId}.historicSql.windowDays from ktx.yaml.`, + }); + } + } + + for (const adapter of config.ingest.adapters) { + const requiredDrivers = ADAPTER_DRIVER_REQUIREMENT[adapter]; + if (!requiredDrivers) continue; + const hasMatching = connections.some((c) => requiredDrivers.includes(c.driver)); + if (!hasMatching) { + warnings.push({ + message: `Adapter "${adapter}" is enabled but no connection of type ${requiredDrivers.slice(0, 2).join('/')} is configured.`, + fix: 'Rerun `ktx setup` to add a connection, or remove the adapter from ingest.adapters.', + }); + } + } + + if (config.agent.run_research.enabled && llm.backend === 'none') { + warnings.push({ + message: 'Research agent is enabled but LLM is not configured.', + fix: 'Set up an LLM provider via `ktx setup` or disable agent.run_research.enabled.', + }); + } + + if (embeddings.backend === 'none' && config.ingest.adapters.includes('live-database')) { + warnings.push({ + message: 'Semantic search is off (embeddings backend = none). Lexical/dictionary lanes still work.', + }); + } + + return warnings; +} + +function buildVerdict( + llm: LlmStatus, + embeddings: EmbeddingsStatus, + connections: ConnectionStatus[], + warnings: WarningItem[], +): { verdict: ProjectVerdict; reason: string; nextActions: string[] } { + if (llm.status === 'fail') { + return { + verdict: 'blocked', + reason: 'LLM not configured — `ktx ask` will not work.', + nextActions: ['ktx setup'], + }; + } + + const reasons: string[] = []; + if (llm.status === 'warn') reasons.push('LLM credentials missing'); + if (embeddings.status === 'warn') { + if (embeddings.backend === 'deterministic' || embeddings.backend === 'none') { + reasons.push('semantic search disabled'); + } else { + reasons.push('embedding credentials missing'); + } + } + const missing = connections.filter((c) => c.status !== 'ok').length; + if (missing > 0) reasons.push(`${missing} connection${missing === 1 ? '' : 's'} need configuration`); + if (warnings.length > 0) reasons.push(`${warnings.length} config warning${warnings.length === 1 ? '' : 's'}`); + + if (reasons.length === 0) { + return { + verdict: 'ready', + reason: 'Ready.', + nextActions: ['ktx scan', 'ktx wiki', 'ktx sl ask "…"'], + }; + } + + return { + verdict: 'partial', + reason: `Partially ready — ${reasons.join('; ')}.`, + nextActions: ['ktx setup'], + }; +} + +export interface BuildProjectStatusOptions { + env?: NodeJS.ProcessEnv; +} + +export function buildProjectStatus(project: KtxLocalProject, options: BuildProjectStatusOptions = {}): ProjectStatus { + const env = options.env ?? process.env; + const config = project.config; + + const llm = buildLlmStatus(config.llm, env); + const embeddings = buildEmbeddingsStatus(config.ingest.embeddings, env); + const storage = buildStorageStatus(config); + const connections = Object.entries(config.connections).map(([name, conn]) => + buildConnectionStatus(name, conn, env), + ); + const pipeline = buildPipelineStatus(config); + const warnings = buildWarnings(config, connections, llm, embeddings); + const { verdict, reason, nextActions } = buildVerdict(llm, embeddings, connections, warnings); + + return { + projectName: config.project, + projectDir: project.projectDir, + llm, + embeddings, + storage, + connections, + pipeline, + warnings, + verdict, + verdictReason: reason, + nextActions, + promptCaching: config.llm.promptCaching + ? { + enabled: config.llm.promptCaching.enabled ?? false, + systemTtl: config.llm.promptCaching.systemTtl, + toolsTtl: config.llm.promptCaching.toolsTtl, + historyTtl: config.llm.promptCaching.historyTtl, + } + : undefined, + workUnits: { + stepBudget: config.ingest.workUnits.stepBudget, + maxConcurrency: config.ingest.workUnits.maxConcurrency, + failureMode: config.ingest.workUnits.failureMode, + }, + memoryAutoCommit: config.memory.auto_commit, + relationshipsDetail: { + acceptThreshold: config.scan.relationships.acceptThreshold, + reviewThreshold: config.scan.relationships.reviewThreshold, + maxLlmTablesPerBatch: config.scan.relationships.maxLlmTablesPerBatch, + validationConcurrency: config.scan.relationships.validationConcurrency, + }, + }; +} + +// ─── Rendering ────────────────────────────────────────────────────────────── + +const SYMBOL: Record = { ok: '✓', warn: '⚠', fail: '✗' }; + +function ansi(useColor: boolean, code: string, text: string, closer = '39'): string { + return useColor ? `\u001b[${code}m${text}\u001b[${closer}m` : text; +} + +function colorFor(level: ProjectStatusLevel): string { + return level === 'ok' ? '32' : level === 'warn' ? '33' : '31'; +} + +function abbreviateHome(filePath: string, env: NodeJS.ProcessEnv): string { + const home = env.HOME; + if (home && (filePath === home || filePath.startsWith(`${home}/`))) { + return filePath === home ? '~' : `~${filePath.slice(home.length)}`; + } + return filePath; +} + +export interface RenderProjectStatusOptions { + verbose?: boolean; + useColor?: boolean; + durationMs?: number; + toolchainChecks?: DoctorCheck[]; + env?: NodeJS.ProcessEnv; +} + +export function renderProjectStatus(status: ProjectStatus, options: RenderProjectStatusOptions = {}): string { + const verbose = options.verbose ?? false; + const useColor = options.useColor ?? false; + const env = options.env ?? process.env; + const dim = (s: string) => ansi(useColor, '2', s, '22'); + const bold = (s: string) => ansi(useColor, '1', s, '22'); + const color = (level: ProjectStatusLevel, s: string) => ansi(useColor, colorFor(level), s); + const sym = (level: ProjectStatusLevel) => color(level, SYMBOL[level]); + + const lines: string[] = []; + const dirStr = abbreviateHome(status.projectDir, env); + lines.push(`${bold('KTX status')} ${dim('·')} ${status.projectName} ${dim(`(${dirStr})`)}`); + lines.push(''); + + const labelPad = 'Connections'.length; + const label = (text: string) => text.padEnd(labelPad); + + // Core readiness rows + const llmDetail = [status.llm.backend, status.llm.model].filter(Boolean).join(` ${dim('·')} `); + lines.push(` ${label('LLM')} ${llmDetail} ${sym(status.llm.status)} ${dim(status.llm.detail)}`); + + const embedParts = [status.embeddings.backend]; + if (status.embeddings.model) embedParts.push(status.embeddings.model); + const embedDim = status.embeddings.dimensions ? `(${status.embeddings.dimensions}d)` : ''; + const embedDetail = `${embedParts.join(` ${dim('·')} `)}${embedDim ? ` ${embedDim}` : ''}`; + lines.push(` ${label('Embeddings')} ${embedDetail} ${sym(status.embeddings.status)} ${dim(status.embeddings.detail)}`); + + lines.push(` ${label('Storage')} ${dim(`${status.storage.state} (state) · ${status.storage.search} (search)`)}`); + lines.push(''); + + // Connections + if (status.connections.length === 0) { + lines.push(` ${bold('Connections')} ${dim('(none)')}`); + lines.push(` ${dim('No connections configured. Run `ktx setup` to add one.')}`); + } else { + lines.push(` ${bold('Connections')} ${dim(`(${status.connections.length})`)}`); + const nameWidth = Math.max(...status.connections.map((c) => c.name.length)); + const driverWidth = Math.max(...status.connections.map((c) => c.driver.length)); + for (const conn of status.connections) { + lines.push( + ` ${sym(conn.status)} ${conn.name.padEnd(nameWidth)} ${dim(conn.driver.padEnd(driverWidth))} ${conn.detail}`, + ); + if (conn.fix && conn.status !== 'ok') { + const indent = 6 + nameWidth + 3 + driverWidth + 3; + lines.push(`${' '.repeat(indent)}${dim(`→ ${conn.fix}`)}`); + } + } + } + lines.push(''); + + // Pipeline + lines.push(` ${bold('Pipeline')}`); + const pipelineLabelWidth = Math.max('Adapters'.length, 'Enrichment'.length, 'Research agent'.length); + const pLabel = (text: string) => text.padEnd(pipelineLabelWidth); + lines.push(` ${pLabel('Adapters')} ${status.pipeline.adapters.length > 0 ? status.pipeline.adapters.join(', ') : dim('(none)')}`); + const enrichmentDetail = [`${status.pipeline.enrichmentMode} mode`]; + if (status.pipeline.relationshipsEnabled) { + const bits = ['relationships on']; + if (status.pipeline.relationshipsLlmProposals) bits.push('LLM proposals'); + if (status.pipeline.relationshipsValidationRequired) bits.push('validation required'); + enrichmentDetail.push(bits.join(', ')); + } else { + enrichmentDetail.push('relationships off'); + } + lines.push(` ${pLabel('Enrichment')} ${enrichmentDetail.join(` ${dim('·')} `)}`); + const agentDetail = status.pipeline.agentEnabled + ? `enabled ${dim(`(${status.pipeline.agentTools.length} tool${status.pipeline.agentTools.length === 1 ? '' : 's'})`)}` + : dim('disabled'); + lines.push(` ${pLabel('Research agent')} ${agentDetail}`); + lines.push(''); + + // Warnings + if (status.warnings.length > 0) { + lines.push(` ${bold('Warnings')}`); + for (const w of status.warnings) { + lines.push(` ${color('warn', SYMBOL.warn)} ${w.message}`); + if (w.fix) lines.push(` ${dim(`→ ${w.fix}`)}`); + } + lines.push(''); + } + + // Verbose extras + if (verbose) { + if (options.toolchainChecks && options.toolchainChecks.length > 0) { + lines.push(` ${bold('Toolchain')}`); + for (const check of options.toolchainChecks) { + const lv: ProjectStatusLevel = check.status === 'pass' ? 'ok' : check.status === 'warn' ? 'warn' : 'fail'; + lines.push(` ${sym(lv)} ${check.label}: ${check.detail}`); + if (check.fix && lv !== 'ok') lines.push(` ${dim(`→ ${check.fix}`)}`); + } + lines.push(''); + } + if (status.promptCaching) { + const pc = status.promptCaching; + const bits = [`enabled=${pc.enabled}`]; + if (pc.systemTtl) bits.push(`system=${pc.systemTtl}`); + if (pc.toolsTtl) bits.push(`tools=${pc.toolsTtl}`); + if (pc.historyTtl) bits.push(`history=${pc.historyTtl}`); + lines.push(` ${bold('Prompt caching')} ${dim(bits.join(', '))}`); + } + if (status.workUnits) { + const wu = status.workUnits; + lines.push(` ${bold('Work units')} ${dim(`stepBudget=${wu.stepBudget}, maxConcurrency=${wu.maxConcurrency}, failureMode=${wu.failureMode}`)}`); + } + if (status.relationshipsDetail) { + const r = status.relationshipsDetail; + lines.push( + ` ${bold('Relationships')} ${dim(`accept=${r.acceptThreshold}, review=${r.reviewThreshold}, maxLlmTables=${r.maxLlmTablesPerBatch}, concurrency=${r.validationConcurrency}`)}`, + ); + } + lines.push( + ` ${bold('Agent')} ${dim(`max_iterations=${status.pipeline.agentMaxIterations}, tools=${status.pipeline.agentTools.join(', ') || '(none)'}`)}`, + ); + lines.push(` ${bold('Memory')} ${dim(`auto_commit=${status.memoryAutoCommit}`)}`); + lines.push( + ` ${bold('Git')} ${dim(`auto_commit=${status.storage.gitAutoCommit}, author=${status.storage.gitAuthor}`)}`, + ); + lines.push(''); + } + + // Verdict + next steps + const verdictLevel: ProjectStatusLevel = + status.verdict === 'ready' ? 'ok' : status.verdict === 'partial' ? 'warn' : 'fail'; + const duration = options.durationMs !== undefined ? ` ${dim(`(${(options.durationMs / 1000).toFixed(2)}s)`)}` : ''; + if (status.verdict === 'ready') { + const hint = ` ${dim('Try:')} ${status.nextActions.join(dim(' · '))}`; + lines.push(`${color(verdictLevel, status.verdictReason)}${hint}${duration}`); + } else { + const hint = status.nextActions.length > 0 ? ` ${dim('Next:')} ${status.nextActions.join(dim(' · '))}` : ''; + lines.push(`${color(verdictLevel, status.verdictReason)}${hint}${duration}`); + } + lines.push(''); + + return lines.join('\n'); +} diff --git a/packages/cli/src/notion-page-picker-tree.test.ts b/packages/cli/src/tree-picker-state.test.ts similarity index 73% rename from packages/cli/src/notion-page-picker-tree.test.ts rename to packages/cli/src/tree-picker-state.test.ts index 94b46b57..52e63f3d 100644 --- a/packages/cli/src/notion-page-picker-tree.test.ts +++ b/packages/cli/src/tree-picker-state.test.ts @@ -11,10 +11,9 @@ import { selectAllVisible, selectNone, toggleChecked, - TRANSIENT_HINT_DURATION_MS, visibleNodeIds, - type NotionPickerPageInput, -} from './notion-page-picker-tree.js'; + type TreePickerNodeInput, +} from './tree-picker-state.js'; const IDS = { engineering: '11111111-1111-1111-1111-111111111111', @@ -28,7 +27,7 @@ const IDS = { cycleB: '99999999-9999-9999-9999-999999999999', }; -function pages(): NotionPickerPageInput[] { +function pages(): TreePickerNodeInput[] { return [ { id: IDS.marketing, title: 'Marketing', archived: false, parentId: null }, { id: IDS.onboarding, title: 'Onboarding', archived: false, parentId: IDS.engineering }, @@ -44,7 +43,7 @@ function pages(): NotionPickerPageInput[] { } describe('buildPickerTree', () => { - it('deduplicates pages, sorts siblings, preserves archived flags, roots orphans, and breaks cycles', () => { + it('deduplicates nodes, sorts siblings, preserves archived flags, roots orphans, and breaks cycles', () => { const tree = buildPickerTree(pages()); const byId = new Map(tree.map((node) => [node.id, node])); @@ -90,8 +89,7 @@ describe('selection invariants', () => { it('checking a parent locks descendants and keeps checked ids minimal', () => { const state = buildInitialState({ tree: buildPickerTree(pages()), - existingRootPageIds: [], - currentCrawlMode: 'selected_roots', + existingSelectedIds: [], }); const checkedParent = toggleChecked(state, IDS.engineering, 1000); @@ -113,15 +111,11 @@ describe('selection invariants', () => { expect(canToggle(IDS.architecture, uncheckedParent)).toEqual({ ok: true }); }); - it('normalizes stored roots, reports stale roots, expands checked ancestors, and flattens descendants', () => { + it('reports stale stored ids via the caller-supplied warning, expands checked ancestors, and flattens descendants', () => { const state = buildInitialState({ tree: buildPickerTree(pages()), - existingRootPageIds: [ - IDS.engineering.replaceAll('-', ''), - IDS.architecture, - 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', - ], - currentCrawlMode: 'selected_roots', + existingSelectedIds: [IDS.engineering, IDS.architecture, 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa'], + staleWarning: (staleCount) => `${staleCount} stored root_page_ids no longer visible`, }); expect([...state.checked]).toEqual([IDS.engineering]); @@ -130,14 +124,21 @@ describe('selection invariants', () => { expect(state.preLoadWarnings).toEqual(['1 stored root_page_ids no longer visible']); expect(flattenSelection(new Set([IDS.engineering, IDS.architecture]), state.byId)).toEqual([IDS.engineering]); }); + + it('falls back to a generic stale warning when no warning factory is supplied', () => { + const state = buildInitialState({ + tree: buildPickerTree(pages()), + existingSelectedIds: ['aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa'], + }); + expect(state.preLoadWarnings).toEqual(['1 stored selections no longer visible']); + }); }); describe('search and cursor movement', () => { it('filters by title and path while deriving auto-expanded ancestors', () => { const state = buildInitialState({ tree: buildPickerTree(pages()), - existingRootPageIds: [], - currentCrawlMode: 'selected_roots', + existingSelectedIds: [], }); const searching = { ...state, @@ -154,8 +155,7 @@ describe('search and cursor movement', () => { it('moves the cursor through visible nodes and implements left/right tree semantics', () => { const state = buildInitialState({ tree: buildPickerTree(pages()), - existingRootPageIds: [], - currentCrawlMode: 'selected_roots', + existingSelectedIds: [], }); const atEngineering = { @@ -176,8 +176,7 @@ describe('bulk actions and reducer effects', () => { it('selects only matching visible roots under search and clears selection', () => { const state = buildInitialState({ tree: buildPickerTree(pages()), - existingRootPageIds: [IDS.marketing], - currentCrawlMode: 'selected_roots', + existingSelectedIds: [IDS.marketing], }); const searching = { ...state, @@ -189,56 +188,56 @@ describe('bulk actions and reducer effects', () => { expect([...selectNone(selected).checked]).toEqual([]); }); - it('returns save immediately for selected_roots and requires confirmation for all_accessible', () => { - const selectedRoots = toggleChecked( + it('saves immediately when confirm is not required and prompts confirmation when requireConfirmOnSave is true', () => { + const noConfirm = toggleChecked( buildInitialState({ tree: buildPickerTree(pages()), - existingRootPageIds: [], - currentCrawlMode: 'selected_roots', + existingSelectedIds: [], }), IDS.marketing, 1000, ); - expect(reducer(selectedRoots, 'save-request')).toEqual({ - next: selectedRoots, + expect(reducer(noConfirm, 'save-request')).toEqual({ + next: noConfirm, effect: 'save', }); - const allAccessible = { - ...selectedRoots, - currentCrawlMode: 'all_accessible' as const, + const confirmRequired = { + ...noConfirm, + requireConfirmOnSave: true, }; - const confirm = reducer(allAccessible, 'save-request'); + const confirm = reducer(confirmRequired, 'save-request'); expect(confirm).toEqual({ - next: { ...allAccessible, pendingConfirm: 'mode-switch' }, + next: { ...confirmRequired, pendingConfirm: 'save-confirm' }, effect: null, }); expect(reducer(confirm.next, 'save-cancel')).toEqual({ - next: { ...allAccessible, pendingConfirm: null }, + next: { ...confirmRequired, pendingConfirm: null }, effect: null, }); expect(reducer(confirm.next, 'save-confirm')).toEqual({ - next: { ...allAccessible, pendingConfirm: null }, + next: { ...confirmRequired, pendingConfirm: null }, effect: 'save', }); }); - it('blocks empty saves, updates search state, and quits without saving', () => { + it('prompts skip-empty confirmation on empty save, updates search state, and quits without saving', () => { const state = buildInitialState({ tree: buildPickerTree(pages()), - existingRootPageIds: [], - currentCrawlMode: 'selected_roots', + existingSelectedIds: [], }); - const blockedSave = reducer(state, 'save-request', 9000); - expect(blockedSave).toEqual({ - next: { - ...state, - transientHint: { - text: 'Select at least one page or press q to quit', - expiresAt: 9000 + TRANSIENT_HINT_DURATION_MS, - }, - }, + const emptySave = reducer(state, 'save-request'); + expect(emptySave).toEqual({ + next: { ...state, pendingConfirm: 'skip-empty' }, + effect: null, + }); + expect(reducer(emptySave.next, 'save-confirm')).toEqual({ + next: { ...state, pendingConfirm: null }, + effect: 'quit-without-save', + }); + expect(reducer(emptySave.next, 'save-cancel')).toEqual({ + next: { ...state, pendingConfirm: null }, effect: null, }); expect( @@ -253,16 +252,33 @@ describe('bulk actions and reducer effects', () => { }); }); + it('treats skip-empty confirmation as a save with empty selection when skipEmptyAction is save-empty', () => { + const state = buildInitialState({ + tree: buildPickerTree(pages()), + existingSelectedIds: [], + skipEmptyAction: 'save-empty', + }); + + const emptySave = reducer(state, 'save-request'); + expect(emptySave).toEqual({ + next: { ...state, pendingConfirm: 'skip-empty' }, + effect: null, + }); + expect(reducer(emptySave.next, 'save-confirm')).toEqual({ + next: { ...state, pendingConfirm: null }, + effect: 'save', + }); + }); + it('clears transient hints only when their expiry time has passed', () => { const state = buildInitialState({ tree: buildPickerTree(pages()), - existingRootPageIds: [], - currentCrawlMode: 'selected_roots', + existingSelectedIds: [], }); const withHint = { ...state, transientHint: { - text: 'Select at least one page or press q to quit', + text: 'Select at least one item or press esc to cancel', expiresAt: 11500, }, }; diff --git a/packages/cli/src/notion-page-picker-tree.ts b/packages/cli/src/tree-picker-state.ts similarity index 86% rename from packages/cli/src/notion-page-picker-tree.ts rename to packages/cli/src/tree-picker-state.ts index 379ac938..9d9b3c68 100644 --- a/packages/cli/src/notion-page-picker-tree.ts +++ b/packages/cli/src/tree-picker-state.ts @@ -1,11 +1,11 @@ -export interface NotionPickerPageInput { +export interface TreePickerNodeInput { id: string; title?: string | null; archived?: boolean; parentId?: string | null; } -interface NotionPickerNode { +export interface TreePickerNode { id: string; title: string; archived: boolean; @@ -15,17 +15,22 @@ interface NotionPickerNode { path: string; } +type PendingConfirmKind = 'save-confirm' | 'skip-empty'; + +export type SkipEmptyAction = 'quit' | 'save-empty'; + export interface PickerState { - tree: NotionPickerNode[]; - byId: Map; + tree: TreePickerNode[]; + byId: Map; expanded: Set; checked: Set; cursorId: string; search: { editing: boolean; query: string }; - pendingConfirm: 'mode-switch' | null; + pendingConfirm: PendingConfirmKind | null; preLoadWarnings: string[]; transientHint: { text: string; expiresAt: number } | null; - currentCrawlMode: 'all_accessible' | 'selected_roots'; + requireConfirmOnSave: boolean; + skipEmptyAction: SkipEmptyAction; } export type PickerCommand = @@ -61,29 +66,16 @@ interface MutableNode { childIds: string[]; } -export const TRANSIENT_HINT_DURATION_MS = 2500; +const TRANSIENT_HINT_DURATION_MS = 2500; const collator = new Intl.Collator('en', { sensitivity: 'base', numeric: true }); -function normalizePageId(value: string): string { - const trimmed = value.trim(); - const compact = trimmed.replace(/-/g, ''); - if (/^[0-9a-fA-F]{32}$/.test(compact)) { - const lower = compact.toLowerCase(); - return `${lower.slice(0, 8)}-${lower.slice(8, 12)}-${lower.slice(12, 16)}-${lower.slice( - 16, - 20, - )}-${lower.slice(20)}`; - } - return trimmed; -} - function titleValue(value: string | null | undefined): string { const trimmed = value?.trim() ?? ''; return trimmed.length > 0 ? trimmed : 'Untitled'; } -function sortedNodeIds(ids: string[], nodes: Map): string[] { +function sortedNodeIds(ids: string[], nodes: Map): string[] { return [...ids].sort((leftId, rightId) => { const left = nodes.get(leftId); const right = nodes.get(rightId); @@ -107,7 +99,7 @@ export function clearExpiredTransientHint(state: PickerState, now = Date.now()): return cloneState(state, { transientHint: null }); } -function ancestorsOf(nodeId: string, byId: Map): string[] { +function ancestorsOf(nodeId: string, byId: Map): string[] { const ancestors: string[] = []; let parentId = byId.get(nodeId)?.parentId ?? null; const seen = new Set(); @@ -119,7 +111,7 @@ function ancestorsOf(nodeId: string, byId: Map): strin return ancestors; } -function descendantsOf(nodeId: string, byId: Map): string[] { +function descendantsOf(nodeId: string, byId: Map): string[] { const result: string[] = []; const stack = [...(byId.get(nodeId)?.childIds ?? [])].reverse(); while (stack.length > 0) { @@ -152,18 +144,18 @@ function matchingIds(state: PickerState): Set { ); } -export function buildPickerTree(searchResults: NotionPickerPageInput[]): NotionPickerNode[] { +export function buildPickerTree(inputs: TreePickerNodeInput[]): TreePickerNode[] { const nodes = new Map(); - for (const result of searchResults) { - const id = normalizePageId(result.id); - if (nodes.has(id)) { + for (const result of inputs) { + const id = result.id.trim(); + if (id.length === 0 || nodes.has(id)) { continue; } nodes.set(id, { id, title: titleValue(result.title), archived: result.archived === true, - parentId: result.parentId ? normalizePageId(result.parentId) : null, + parentId: result.parentId ? result.parentId.trim() : null, childIds: [], }); } @@ -202,7 +194,7 @@ export function buildPickerTree(searchResults: NotionPickerPageInput[]): NotionP [...nodes.values()].filter((node) => node.parentId === null).map((node) => node.id), nodes, ); - const tree: NotionPickerNode[] = []; + const tree: TreePickerNode[] = []; function visit(nodeId: string, depth: number, pathPrefix: string[]): void { const raw = nodes.get(nodeId); @@ -210,7 +202,7 @@ export function buildPickerTree(searchResults: NotionPickerPageInput[]): NotionP return; } const path = [...pathPrefix, raw.title].join(' / '); - const node: NotionPickerNode = { + const node: TreePickerNode = { id: raw.id, title: raw.title, archived: raw.archived, @@ -232,11 +224,11 @@ export function buildPickerTree(searchResults: NotionPickerPageInput[]): NotionP return tree; } -export function isAncestorChecked(nodeId: string, checked: Set, byId: Map): boolean { +export function isAncestorChecked(nodeId: string, checked: Set, byId: Map): boolean { return ancestorsOf(nodeId, byId).some((ancestorId) => checked.has(ancestorId)); } -function checkedAncestor(nodeId: string, state: PickerState): NotionPickerNode | null { +function checkedAncestor(nodeId: string, state: PickerState): TreePickerNode | null { for (const ancestorId of ancestorsOf(nodeId, state.byId)) { if (state.checked.has(ancestorId)) { return state.byId.get(ancestorId) ?? null; @@ -247,7 +239,7 @@ function checkedAncestor(nodeId: string, state: PickerState): NotionPickerNode | export function canToggle(nodeId: string, state: PickerState): { ok: true } | { ok: false; reason: string } { if (!state.byId.has(nodeId)) { - return { ok: false, reason: 'Page not found' }; + return { ok: false, reason: 'Node not found' }; } const ancestor = checkedAncestor(nodeId, state); if (ancestor) { @@ -276,7 +268,7 @@ export function toggleChecked(state: PickerState, nodeId: string, now = Date.now return cloneState(state, { checked, transientHint: null }); } -export function flattenSelection(checked: Set, byId: Map): string[] { +export function flattenSelection(checked: Set, byId: Map): string[] { const result: string[] = []; for (const node of byId.values()) { if (checked.has(node.id) && !isAncestorChecked(node.id, checked, byId)) { @@ -402,16 +394,21 @@ export function moveCursor(state: PickerState, dir: 'up' | 'down' | 'left' | 'ri } export function buildInitialState(args: { - tree: NotionPickerNode[]; - existingRootPageIds: string[]; - currentCrawlMode?: 'all_accessible' | 'selected_roots'; + tree: TreePickerNode[]; + existingSelectedIds: string[]; + requireConfirmOnSave?: boolean; + skipEmptyAction?: SkipEmptyAction; + staleWarning?: (staleCount: number) => string; }): PickerState { const byId = new Map(args.tree.map((node) => [node.id, node])); const checked = new Set(); let staleCount = 0; - for (const rawId of args.existingRootPageIds) { - const id = normalizePageId(rawId); + for (const rawId of args.existingSelectedIds) { + const id = rawId.trim(); + if (id.length === 0) { + continue; + } if (byId.has(id)) { checked.add(id); } else { @@ -427,6 +424,12 @@ export function buildInitialState(args: { } } + const preLoadWarnings: string[] = []; + if (staleCount > 0) { + const warning = args.staleWarning ? args.staleWarning(staleCount) : `${staleCount} stored selections no longer visible`; + preLoadWarnings.push(warning); + } + return { tree: args.tree, byId, @@ -435,16 +438,19 @@ export function buildInitialState(args: { cursorId: args.tree[0]?.id ?? '', search: { editing: false, query: '' }, pendingConfirm: null, - preLoadWarnings: staleCount > 0 ? [`${staleCount} stored root_page_ids no longer visible`] : [], + preLoadWarnings, transientHint: null, - currentCrawlMode: args.currentCrawlMode ?? 'selected_roots', + requireConfirmOnSave: args.requireConfirmOnSave ?? false, + skipEmptyAction: args.skipEmptyAction ?? 'quit', }; } export function reducer(state: PickerState, cmd: PickerCommand, now = Date.now()): { next: PickerState; effect: PickerEffect } { if (state.pendingConfirm) { if (cmd === 'save-confirm') { - return { next: cloneState(state, { pendingConfirm: null }), effect: 'save' }; + const effect: PickerEffect = + state.pendingConfirm === 'skip-empty' ? (state.skipEmptyAction === 'save-empty' ? 'save' : 'quit-without-save') : 'save'; + return { next: cloneState(state, { pendingConfirm: null }), effect }; } if (cmd === 'save-cancel') { return { next: cloneState(state, { pendingConfirm: null }), effect: null }; @@ -498,19 +504,13 @@ export function reducer(state: PickerState, cmd: PickerCommand, now = Date.now() }; case 'save-request': if (state.checked.size === 0) { - return { - next: cloneState(state, { - transientHint: transientHint('Select at least one page or press q to quit', now), - }), - effect: null, - }; + return { next: cloneState(state, { pendingConfirm: 'skip-empty' }), effect: null }; } - if (state.currentCrawlMode === 'all_accessible') { - return { next: cloneState(state, { pendingConfirm: 'mode-switch' }), effect: null }; + if (state.requireConfirmOnSave) { + return { next: cloneState(state, { pendingConfirm: 'save-confirm' }), effect: null }; } return { next: state, effect: 'save' }; case 'save-confirm': - return { next: state, effect: 'save' }; case 'save-cancel': return { next: state, effect: null }; case 'quit': diff --git a/packages/cli/src/tree-picker-tui.test.tsx b/packages/cli/src/tree-picker-tui.test.tsx new file mode 100644 index 00000000..8c4f8d1e --- /dev/null +++ b/packages/cli/src/tree-picker-tui.test.tsx @@ -0,0 +1,361 @@ +/* @jsxImportSource react */ +import { render as renderInkTest } from 'ink-testing-library'; +import { type ReactNode } from 'react'; +import { describe, expect, it, vi } from 'vitest'; +import { buildInitialState, buildPickerTree, type TreePickerNodeInput } from './tree-picker-state.js'; +import { + TreePickerApp, + renderTreePickerTui, + resolveTreePickerWidth, + sanitizeTreePickerTuiError, + treePickerCommandForInkInput, + windowItems, + windowOffset, + type TreePickerChrome, + type TreePickerInkInstance, + type TreePickerInkRenderOptions, +} from './tree-picker-tui.js'; + +const IDS = { + engineering: '11111111-1111-1111-1111-111111111111', + architecture: '22222222-2222-2222-2222-222222222222', + marketing: '33333333-3333-3333-3333-333333333333', + finance: '44444444-4444-4444-4444-444444444444', + ops: '55555555-5555-5555-5555-555555555555', + sales: '66666666-6666-6666-6666-666666666666', + support: '77777777-7777-7777-7777-777777777777', + product: '88888888-8888-8888-8888-888888888888', + design: '99999999-9999-9999-9999-999999999999', +}; + +function pages(): TreePickerNodeInput[] { + return [ + { id: IDS.engineering, title: 'Engineering Docs', archived: false, parentId: null }, + { id: IDS.architecture, title: 'Architecture', archived: false, parentId: IDS.engineering }, + { id: IDS.marketing, title: 'Marketing', archived: false, parentId: null }, + ]; +} + +function manyPages(): TreePickerNodeInput[] { + return [ + { id: IDS.engineering, title: 'Engineering Docs', archived: false, parentId: null }, + { id: IDS.architecture, title: 'Architecture', archived: false, parentId: IDS.engineering }, + { id: IDS.marketing, title: 'Marketing', archived: false, parentId: null }, + { id: IDS.finance, title: 'Finance', archived: false, parentId: null }, + { id: IDS.ops, title: 'Operations', archived: false, parentId: null }, + { id: IDS.sales, title: 'Sales', archived: false, parentId: null }, + { id: IDS.support, title: 'Support', archived: false, parentId: null }, + { id: IDS.product, title: 'Product', archived: false, parentId: null }, + { id: IDS.design, title: 'Design', archived: false, parentId: null }, + ]; +} + +function state(options: { requireConfirmOnSave?: boolean } = {}) { + return buildInitialState({ + tree: buildPickerTree(pages()), + existingSelectedIds: [], + requireConfirmOnSave: options.requireConfirmOnSave ?? false, + }); +} + +function chrome(overrides: Partial = {}): TreePickerChrome { + return { + title: 'Select items', + subtitleLines: ['Source: Test'], + ...overrides, + }; +} + +async function waitForInkInput(): Promise { + await new Promise((resolve) => setTimeout(resolve, 10)); +} + +function fakeInkInstance(): TreePickerInkInstance { + return { + rerender: vi.fn(), + unmount: vi.fn(), + waitUntilExit: vi.fn(async () => undefined), + }; +} + +function normalizeFrameWrap(frame: string | undefined): string { + return frame?.replace(/\n/g, ' ').replace(/│ /g, '').replace(/ +/g, ' ') ?? ''; +} + +describe('treePickerCommandForInkInput', () => { + it('maps browse, search, and confirm input to reducer commands', () => { + expect(treePickerCommandForInkInput('', { downArrow: true }, state().search, null)).toBe('cursor-down'); + expect(treePickerCommandForInkInput('', { upArrow: true }, state().search, null)).toBe('cursor-up'); + expect(treePickerCommandForInkInput('', { rightArrow: true }, state().search, null)).toBe('cursor-right'); + expect(treePickerCommandForInkInput('', { leftArrow: true }, state().search, null)).toBe('cursor-left'); + expect(treePickerCommandForInkInput(' ', {}, state().search, null)).toBe('toggle-check'); + expect(treePickerCommandForInkInput('/', {}, state().search, null)).toBe('search-start'); + expect(treePickerCommandForInkInput('a', {}, state().search, null)).toBe('select-all-visible'); + expect(treePickerCommandForInkInput('n', {}, state().search, null)).toBe('select-none'); + expect(treePickerCommandForInkInput('', { return: true }, state().search, null)).toBe('save-request'); + expect(treePickerCommandForInkInput('', { escape: true }, state().search, null)).toBe('quit'); + expect(treePickerCommandForInkInput('c', { ctrl: true }, state().search, null)).toBe('quit'); + expect(treePickerCommandForInkInput('s', {}, state().search, null)).toBeNull(); + expect(treePickerCommandForInkInput('q', {}, state().search, null)).toBeNull(); + + expect(treePickerCommandForInkInput('x', {}, { editing: true, query: '' }, null)).toEqual({ + type: 'search-input', + value: 'x', + }); + expect(treePickerCommandForInkInput('', { backspace: true }, { editing: true, query: 'x' }, null)).toBe( + 'search-backspace', + ); + expect(treePickerCommandForInkInput('', { return: true }, { editing: true, query: 'x' }, null)).toBe( + 'search-submit', + ); + expect(treePickerCommandForInkInput('', { escape: true }, { editing: true, query: 'x' }, null)).toBe( + 'search-cancel', + ); + + expect(treePickerCommandForInkInput('y', {}, state().search, 'save-confirm')).toBe('save-confirm'); + expect(treePickerCommandForInkInput('', { return: true }, state().search, 'save-confirm')).toBe('save-confirm'); + expect(treePickerCommandForInkInput('n', {}, state().search, 'save-confirm')).toBe('save-cancel'); + }); +}); + +describe('window helpers', () => { + it('centers the selected row and returns the visible slice', () => { + expect(windowOffset(20, 10, 5)).toBe(8); + expect(windowItems(['a', 'b', 'c', 'd', 'e'], 3, 3)).toEqual({ items: ['c', 'd', 'e'], offset: 2 }); + }); + + it('clamps picker width to the design rule', () => { + expect(resolveTreePickerWidth(200)).toBe(120); + expect(resolveTreePickerWidth(100)).toBe(96); + expect(resolveTreePickerWidth(50)).toBe(60); + expect(resolveTreePickerWidth(undefined)).toBe(96); + }); +}); + +describe('TreePickerApp', () => { + it('renders chrome title, subtitle, warnings, help, and row glyphs', () => { + const initialState = { + ...state(), + preLoadWarnings: ['1 stale stored selections - they will be removed if you save'], + }; + const { lastFrame } = renderInkTest( + , + ); + + const frame = lastFrame() ?? ''; + expect(frame).toContain('Select fancy widgets'); + expect(frame).toContain('Workspace: Design Workspace'); + expect(frame).toContain('5000-page cap reached - some pages not shown'); + expect(frame).toContain('1 stale stored selections - they will be removed if you save'); + expect(frame).toContain('◻ Engineering Docs ▸ (1)'); + expect(frame).toContain('◻ Marketing'); + expect(normalizeFrameWrap(frame)).toContain( + 'Right Arrow to expand, Up/Down to move, Space to select or unselect, Slash to filter, Enter to confirm, Escape to go back, or Ctrl+C to exit.', + ); + }); + + it('renders custom help text when supplied', () => { + const { lastFrame } = renderInkTest( + , + ); + expect(lastFrame() ?? '').toContain('Bespoke instructions here.'); + }); + + it('renders checked parents and locked descendants with locked glyphs', () => { + const initialState = { + ...state(), + checked: new Set([IDS.engineering]), + expanded: new Set([IDS.engineering]), + }; + const { lastFrame } = renderInkTest( + , + ); + + const frame = lastFrame() ?? ''; + expect(frame).toContain('◼ Engineering Docs ▾'); + expect(frame).toContain(' ◼ Architecture'); + }); + + it('supports keyboard selection, confirm-on-save, and save callback', async () => { + const onExit = vi.fn(); + const { stdin, lastFrame } = renderInkTest( + + `Confirm: ${current.checked.size} item${current.checked.size === 1 ? '' : 's'}? Press Enter or Escape.`, + })} + terminalRows={24} + terminalWidth={100} + onExit={onExit} + />, + ); + + stdin.write(' '); + await waitForInkInput(); + expect(lastFrame()).toContain('◼ Engineering Docs'); + + stdin.write('\r'); + await waitForInkInput(); + expect(normalizeFrameWrap(lastFrame())).toContain('Confirm: 1 item? Press Enter or Escape.'); + + stdin.write('y'); + await waitForInkInput(); + expect(onExit).toHaveBeenCalledWith({ kind: 'save', selectedIds: [IDS.engineering] }); + }); + + it('uses the chrome-supplied skip-empty message and quits on confirm', async () => { + const onExit = vi.fn(); + const { stdin, lastFrame } = renderInkTest( + , + ); + + stdin.write('\r'); + await waitForInkInput(); + expect(normalizeFrameWrap(lastFrame())).toContain('No selections. Skip or back?'); + expect(onExit).not.toHaveBeenCalled(); + + stdin.write('n'); + await waitForInkInput(); + expect(lastFrame()).not.toContain('No selections. Skip or back?'); + expect(onExit).not.toHaveBeenCalled(); + + stdin.write('\r'); + await waitForInkInput(); + expect(lastFrame()).toContain('No selections. Skip or back?'); + + stdin.write('\r'); + await waitForInkInput(); + expect(onExit).toHaveBeenCalledWith({ kind: 'quit' }); + }); + + it('renders row-window overflow indicators when the visible list is clipped', async () => { + const onExit = vi.fn(); + const initialState = buildInitialState({ + tree: buildPickerTree(manyPages()), + existingSelectedIds: [], + }); + initialState.expanded = new Set([IDS.engineering]); + const { stdin, lastFrame } = renderInkTest( + , + ); + + expect(lastFrame()).toContain('↓ 4 more'); + + stdin.write(''); + stdin.write(''); + stdin.write(''); + stdin.write(''); + await waitForInkInput(); + + const frame = lastFrame() ?? ''; + expect(frame).toContain('↑ '); + expect(frame).toContain('↓ '); + expect(onExit).not.toHaveBeenCalled(); + }); + + it('quits without saving on Ctrl+C', async () => { + const onExit = vi.fn(); + const { stdin } = renderInkTest( + , + ); + + stdin.write(''); + await waitForInkInput(); + expect(onExit).toHaveBeenCalledWith({ kind: 'quit' }); + }); +}); + +describe('renderTreePickerTui', () => { + it('returns the app result from the Ink runtime', async () => { + const io = { + stdin: { isTTY: true, setRawMode: vi.fn() }, + stdout: { isTTY: true, columns: 100, rows: 24, write: vi.fn() }, + stderr: { write: vi.fn() }, + }; + const renderInk = vi.fn((_tree: ReactNode, _options: TreePickerInkRenderOptions) => fakeInkInstance()); + + await expect( + renderTreePickerTui( + { initialState: state(), chrome: chrome() }, + io, + { renderInk }, + ), + ).resolves.toEqual({ kind: 'quit' }); + expect(renderInk).toHaveBeenCalledOnce(); + }); + + it('sanitizes render errors and uses the supplied scripted-mode hint', async () => { + expect(sanitizeTreePickerTuiError(new Error('token=secret https://api.example.com/v1/search'))).toBe( + '[redacted] [redacted-url]', + ); + }); + + it('falls back to quit with the scripted-mode hint when Ink cannot initialize', async () => { + let stderr = ''; + const io = { + stdin: { isTTY: false, setRawMode: vi.fn() }, + stdout: { isTTY: false, columns: 100, rows: 24, write: vi.fn() }, + stderr: { + write(chunk: string) { + stderr += chunk; + }, + }, + }; + + await expect( + renderTreePickerTui( + { initialState: state(), chrome: chrome() }, + io, + { + renderInk: vi.fn(() => { + throw new Error('token=secret'); + }), + scriptedModeHint: 'Use --no-input --foo bar for scripted mode.', + }, + ), + ).resolves.toEqual({ kind: 'quit' }); + expect(stderr).toContain('Use --no-input --foo bar for scripted mode.'); + expect(stderr).not.toContain('secret'); + }); +}); diff --git a/packages/cli/src/notion-page-picker-tui.tsx b/packages/cli/src/tree-picker-tui.tsx similarity index 56% rename from packages/cli/src/notion-page-picker-tui.tsx rename to packages/cli/src/tree-picker-tui.tsx index 30af7522..9cdbef8d 100644 --- a/packages/cli/src/notion-page-picker-tui.tsx +++ b/packages/cli/src/tree-picker-tui.tsx @@ -9,13 +9,14 @@ import { visibleNodeIds, type PickerCommand, type PickerState, -} from './notion-page-picker-tree.js'; +} from './tree-picker-state.js'; import type { KtxCliIo } from './cli-runtime.js'; const COLOR_THEME = { text: 'white', muted: 'gray', active: 'cyan', + selected: 'green', warning: 'yellow', } as const; @@ -23,12 +24,19 @@ const NO_COLOR_THEME = { text: 'white', muted: 'white', active: 'white', + selected: 'white', warning: 'white', } as const; -type NotionPickerTheme = Record; +type TreePickerTheme = Record; -export interface NotionPickerTuiIo extends KtxCliIo { +const DEFAULT_TREE_PICKER_HELP_TEXT = + 'Right Arrow to expand, Up/Down to move, Space to select or unselect, Slash to filter, Enter to confirm, Escape to go back, or Ctrl+C to exit.'; + +const DEFAULT_SKIP_EMPTY_MESSAGE = + 'Nothing selected. Skip this step? Press Enter to skip or Escape to go back.'; + +export interface TreePickerTuiIo extends KtxCliIo { stdin?: { isTTY?: boolean; setRawMode?(value: boolean): void }; stdout: KtxCliIo['stdout'] & { isTTY?: boolean; columns?: number; rows?: number }; } @@ -45,58 +53,54 @@ interface InkKey { delete?: boolean; } -export type PickerRenderResult = { kind: 'save'; rootPageIds: string[] } | { kind: 'quit' }; +export type TreePickerResult = { kind: 'save'; selectedIds: string[] } | { kind: 'quit' }; -export interface PickerRenderInput { - initialState: PickerState; - connectionId: string; - workspaceLabel: string; - cappedAtCount: number | null; - currentCrawlMode: 'all_accessible' | 'selected_roots'; +export interface TreePickerChrome { + title: string; + helpText?: string; + subtitleLines?: readonly string[]; + warningLines?: readonly string[]; + confirmSaveMessage?: (state: PickerState) => string; + skipEmptyMessage?: string; } -interface NotionPickerAppProps extends PickerRenderInput { +export interface TreePickerRenderInput { + initialState: PickerState; + chrome: TreePickerChrome; +} + +interface TreePickerAppProps extends TreePickerRenderInput { terminalRows?: number; terminalWidth?: number; env?: NodeJS.ProcessEnv; - onExit(result: PickerRenderResult): void; + onExit(result: TreePickerResult): void; } -export interface NotionPickerInkInstance { +export interface TreePickerInkInstance { rerender(tree: ReactNode): void; unmount(): void; waitUntilExit(): Promise; } -export interface NotionPickerInkRenderOptions { - stdin?: NotionPickerTuiIo['stdin']; - stdout: NotionPickerTuiIo['stdout']; - stderr: NotionPickerTuiIo['stderr']; +export interface TreePickerInkRenderOptions { + stdin?: TreePickerTuiIo['stdin']; + stdout: TreePickerTuiIo['stdout']; + stderr: TreePickerTuiIo['stderr']; exitOnCtrlC: boolean; patchConsole: boolean; maxFps: number; alternateScreen: boolean; } -function resolveTheme(env: NodeJS.ProcessEnv = process.env): NotionPickerTheme { +function resolveTheme(env: NodeJS.ProcessEnv = process.env): TreePickerTheme { return env.NO_COLOR || env.TERM === 'dumb' ? NO_COLOR_THEME : COLOR_THEME; } -export function resolveNotionPickerWidth(columns: number | undefined): number { +export function resolveTreePickerWidth(columns: number | undefined): number { const resolvedColumns = columns ?? 100; return Math.max(60, Math.min(120, resolvedColumns - 4)); } -function staleWarningText(warning: string): string { - return warning.includes('stored root_page_ids no longer visible') - ? `${warning} - they will be removed if you save` - : warning; -} - -function selectedPageCountText(count: number): string { - return `${count} selected ${count === 1 ? 'page' : 'pages'}`; -} - function rowMatchesSearch(state: PickerState, nodeId: string): boolean { const query = state.search.query.trim().toLocaleLowerCase(); if (!query) { @@ -109,7 +113,7 @@ function rowMatchesSearch(state: PickerState, nodeId: string): boolean { return node.title.toLocaleLowerCase().includes(query) || node.path.toLocaleLowerCase().includes(query); } -export function sanitizeNotionPickerTuiError(error: unknown): string { +export function sanitizeTreePickerTuiError(error: unknown): string { const message = error instanceof Error ? error.message : String(error); return message .replace(/[a-z][a-z0-9+.-]*:\/\/[^\s]+/gi, '[redacted-url]') @@ -132,7 +136,7 @@ function truncateText(value: string, width: number): string { return `${value.slice(0, width - 3)}...`; } -export function notionPickerCommandForInkInput( +export function treePickerCommandForInkInput( input: string, key: InkKey, search: PickerState['search'], @@ -150,7 +154,7 @@ export function notionPickerCommandForInkInput( if (key.backspace || key.delete) return 'search-backspace'; if (key.downArrow) return 'cursor-down'; if (key.upArrow) return 'cursor-up'; - if (input.length === 1 && input >= ' ' && input !== '\u007f') return { type: 'search-input', value: input }; + if (input.length === 1 && input >= ' ' && input !== '') return { type: 'search-input', value: input }; return null; } if (key.ctrl === true && input === 'c') return 'quit'; @@ -158,54 +162,64 @@ export function notionPickerCommandForInkInput( if (key.downArrow) return 'cursor-down'; if (key.leftArrow) return 'cursor-left'; if (key.rightArrow) return 'cursor-right'; - if (key.return) return 'expand'; + if (key.return) return 'save-request'; if (input === ' ') return 'toggle-check'; if (input === '/') return 'search-start'; if (input === 'a') return 'select-all-visible'; if (input === 'n') return 'select-none'; - if (input === 's') return 'save-request'; - if (input === 'q' || key.escape) return 'quit'; + if (key.escape) return 'quit'; return null; } -function PickerRow(props: { state: PickerState; nodeId: string; width: number; theme: NotionPickerTheme }): ReactNode { +function PickerRow(props: { state: PickerState; nodeId: string; width: number; theme: TreePickerTheme }): ReactNode { const node = props.state.byId.get(props.nodeId); if (!node) return null; const focused = props.state.cursorId === node.id; const locked = isAncestorChecked(node.id, props.state.checked, props.state.byId); const checked = props.state.checked.has(node.id); - const glyph = locked ? '[~]' : checked ? '[×]' : '[ ]'; - const children = + const isSelected = checked || locked; + const glyph = isSelected ? '◼' : '◻'; + const glyphColor = checked || locked ? props.theme.selected : props.theme.muted; + const childAffordance = node.childIds.length > 0 ? (props.state.expanded.has(node.id) ? ' ▾' : ` ▸ (${node.childIds.length})`) : ''; - const prefix = `${focused ? '▸' : ' '} ${glyph} ${' '.repeat(node.depth * 2)}`; - const color = focused ? props.theme.active : locked || node.archived ? props.theme.muted : props.theme.text; - const title = truncateText(`${node.title}${children}`, Math.max(10, props.width - prefix.length)); + const indent = ' '.repeat(node.depth * 2); + const titleColor = focused ? props.theme.active : props.theme.text; const inverse = rowMatchesSearch(props.state, node.id); + const prefixWidth = indent.length + 2 + childAffordance.length; + const title = truncateText(node.title, Math.max(10, props.width - prefixWidth)); return ( - - {prefix} - {title} + + + {indent} + {glyph} + + + {' '} + {title} + + {childAffordance.length > 0 ? {childAffordance} : null} ); } -export function NotionPickerApp(props: NotionPickerAppProps): ReactNode { +export function TreePickerApp(props: TreePickerAppProps): ReactNode { const app = useApp(); const [state, setState] = useState(props.initialState); const stateRef = useRef(state); const theme = useMemo(() => resolveTheme(props.env), [props.env]); const visibleIds = visibleNodeIds(state); const selectedIndex = Math.max(0, visibleIds.indexOf(state.cursorId)); - const reservedRows = state.pendingConfirm === 'mode-switch' ? 9 : 8; - const visibleRows = Math.max(5, Math.min(20, (props.terminalRows ?? 24) - reservedRows)); + const reservedRows = state.pendingConfirm === 'save-confirm' ? 10 : 9; + const visibleRows = Math.max(5, Math.min(12, (props.terminalRows ?? 24) - reservedRows)); const rows = windowItems(visibleIds, selectedIndex, visibleRows); const hiddenAbove = rows.offset; const hiddenBelow = Math.max(0, visibleIds.length - rows.offset - rows.items.length); const searchMatchCount = filterTree(state).visibleIds.size; - const width = resolveNotionPickerWidth(props.terminalWidth); + const width = resolveTreePickerWidth(props.terminalWidth); const showSearch = state.search.editing || state.search.query.trim().length > 0; - const selectedCount = flattenSelection(state.checked, state.byId).length; + const helpText = props.chrome.helpText ?? DEFAULT_TREE_PICKER_HELP_TEXT; + const skipEmptyMessage = props.chrome.skipEmptyMessage ?? DEFAULT_SKIP_EMPTY_MESSAGE; stateRef.current = state; @@ -234,7 +248,7 @@ export function NotionPickerApp(props: NotionPickerAppProps): ReactNode { }, [state.transientHint?.expiresAt]); useInput((input, key) => { - const command = notionPickerCommandForInkInput(input, key, stateRef.current.search, stateRef.current.pendingConfirm); + const command = treePickerCommandForInkInput(input, key, stateRef.current.search, stateRef.current.pendingConfirm); if (!command) { return; } @@ -242,7 +256,7 @@ export function NotionPickerApp(props: NotionPickerAppProps): ReactNode { stateRef.current = next; setState(next); if (effect === 'save') { - props.onExit({ kind: 'save', rootPageIds: flattenSelection(next.checked, next.byId) }); + props.onExit({ kind: 'save', selectedIds: flattenSelection(next.checked, next.byId) }); app.exit(); return; } @@ -254,39 +268,68 @@ export function NotionPickerApp(props: NotionPickerAppProps): ReactNode { return ( - Notion pages visible to integration "{props.workspaceLabel}" - {props.cappedAtCount ? {props.cappedAtCount}-page cap reached - some pages not shown : null} - {state.preLoadWarnings.map((warning) => ( - - {staleWarningText(warning)} - - ))} - {showSearch ? ( - - / {state.search.query} - {state.search.editing ? '█' : ''} ({searchMatchCount} matches) - - ) : null} - + + ◆ + {props.chrome.title} + + + {helpText} + + {(props.chrome.subtitleLines ?? []).map((line, idx) => ( + + {line} + + ))} + {(props.chrome.warningLines ?? []).map((line, idx) => ( + + {line} + + ))} + {state.preLoadWarnings.map((warning) => ( + + {warning} + + ))} + {showSearch ? ( + + / + + {state.search.query} + {state.search.editing ? '█' : ''} + + ({searchMatchCount} matches) + + ) : null} + {hiddenAbove > 0 ? ↑ {hiddenAbove} more : null} {rows.items.map((nodeId) => ( ))} {hiddenBelow > 0 ? ↓ {hiddenBelow} more : null} + {state.pendingConfirm === 'save-confirm' ? ( + + {props.chrome.confirmSaveMessage + ? props.chrome.confirmSaveMessage(state) + : 'Confirm save? Press Enter to confirm or Escape to go back.'} + + ) : null} + {state.pendingConfirm === 'skip-empty' ? {skipEmptyMessage} : null} + {state.transientHint ? {state.transientHint.text} : null} - {state.pendingConfirm === 'mode-switch' ? ( - - Save will switch crawl_mode all_accessible -> selected_roots and limit ingest to{' '} - {selectedPageCountText(selectedCount)}. [y] confirm [esc] back - - ) : null} - {state.transientHint ? {state.transientHint.text} : null} - space toggle · enter expand · / search · a all · n none · s save & exit · q quit + └ ); } -function renderInk(tree: ReactNode, options: NotionPickerInkRenderOptions): NotionPickerInkInstance { +function renderInk(tree: ReactNode, options: TreePickerInkRenderOptions): TreePickerInkInstance { return renderInkRuntime(tree, { stdin: options.stdin as NodeJS.ReadStream | undefined, stdout: options.stdout as NodeJS.WriteStream, @@ -295,19 +338,24 @@ function renderInk(tree: ReactNode, options: NotionPickerInkRenderOptions): Noti patchConsole: options.patchConsole, maxFps: options.maxFps, alternateScreen: options.alternateScreen, - }) as NotionPickerInkInstance; + }) as TreePickerInkInstance; } -export async function renderNotionPickerTui( - input: PickerRenderInput, - io: NotionPickerTuiIo, - options: { renderInk?: (tree: ReactNode, options: NotionPickerInkRenderOptions) => NotionPickerInkInstance } = {}, -): Promise { - let result: PickerRenderResult = { kind: 'quit' }; - let instance: NotionPickerInkInstance | null = null; +export interface RenderTreePickerOptions { + renderInk?: (tree: ReactNode, options: TreePickerInkRenderOptions) => TreePickerInkInstance; + scriptedModeHint?: string; +} + +export async function renderTreePickerTui( + input: TreePickerRenderInput, + io: TreePickerTuiIo, + options: RenderTreePickerOptions = {}, +): Promise { + let result: TreePickerResult = { kind: 'quit' }; + let instance: TreePickerInkInstance | null = null; try { instance = (options.renderInk ?? renderInk)( - for scripted mode. ${sanitizeNotionPickerTuiError(error)}\n`, - ); + const hint = options.scriptedModeHint ?? 'Picker requires a TTY.'; + io.stderr.write(`${hint} ${sanitizeTreePickerTuiError(error)}\n`); return { kind: 'quit' }; } } diff --git a/packages/context/src/ingest/adapters/historic-sql/redaction.test.ts b/packages/context/src/ingest/adapters/historic-sql/redaction.test.ts index c8f1d78b..d27015a6 100644 --- a/packages/context/src/ingest/adapters/historic-sql/redaction.test.ts +++ b/packages/context/src/ingest/adapters/historic-sql/redaction.test.ts @@ -9,7 +9,7 @@ describe('historic-SQL redaction', () => { ]); const sql = - "select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; + "select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; // pragma: allowlist secret expect(redactHistoricSqlText(sql, redactors)).toBe( "select * from public.api_events where api_key = '[REDACTED]' and note = '[REDACTED]'", diff --git a/packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts b/packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts index 421970bf..d1610054 100644 --- a/packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts +++ b/packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts @@ -169,7 +169,7 @@ describe('stageHistoricSqlAggregatedSnapshot', () => { it('redacts configured SQL substrings in staged artifacts while analyzing original SQL', async () => { const stagedDir = await tempDir(); const originalSql = - "select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; + "select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; // pragma: allowlist secret const reader: HistoricSqlReader = { async probe() { return { warnings: [], info: [] }; diff --git a/packages/context/src/ingest/adapters/metabase/client.test.ts b/packages/context/src/ingest/adapters/metabase/client.test.ts index 1c0fdfa9..3d45a276 100644 --- a/packages/context/src/ingest/adapters/metabase/client.test.ts +++ b/packages/context/src/ingest/adapters/metabase/client.test.ts @@ -92,7 +92,7 @@ describe('MetabaseClient retry exhaustion', () => { .mockResolvedValueOnce(new Response(JSON.stringify([]), { status: 200 })); const client = new MetabaseClient( - { apiUrl: 'https://metabase.example.test', apiKey: 'key' }, + { apiUrl: 'https://metabase.example.test', apiKey: 'key' }, // pragma: allowlist secret { ...DEFAULT_METABASE_CLIENT_CONFIG, baseDelayMs: 0, diff --git a/packages/context/src/ingest/adapters/metabase/local-metabase.adapter.test.ts b/packages/context/src/ingest/adapters/metabase/local-metabase.adapter.test.ts index 7cbe913b..b25ea18b 100644 --- a/packages/context/src/ingest/adapters/metabase/local-metabase.adapter.test.ts +++ b/packages/context/src/ingest/adapters/metabase/local-metabase.adapter.test.ts @@ -39,7 +39,7 @@ describe('metabaseRuntimeConfigFromLocalConnection', () => { const connection: KtxProjectConnectionConfig = { driver: 'metabase', api_url: 'https://metabase.example.com', - api_key_ref: `file:${keyPath}`, + api_key_ref: `file:${keyPath}`, // pragma: allowlist secret }; expect(metabaseRuntimeConfigFromLocalConnection('prod-metabase', connection)).toEqual({ diff --git a/packages/llm/src/embedding-health.test.ts b/packages/llm/src/embedding-health.test.ts index ca998aa9..65956311 100644 --- a/packages/llm/src/embedding-health.test.ts +++ b/packages/llm/src/embedding-health.test.ts @@ -17,13 +17,13 @@ describe('KTX embedding health check', () => { backend: 'openai', model: 'text-embedding-3-small', dimensions: 3, - openai: { apiKey: 'sk-openai-test' }, + openai: { apiKey: 'sk-openai-test' }, // pragma: allowlist secret }, { deps: { createOpenAIClient } }, ), ).resolves.toEqual({ ok: true }); - expect(createOpenAIClient).toHaveBeenCalledWith({ apiKey: 'sk-openai-test', baseURL: undefined }); + expect(createOpenAIClient).toHaveBeenCalledWith({ apiKey: 'sk-openai-test', baseURL: undefined }); // pragma: allowlist secret }); it('returns failed when the provider returns the wrong dimensions', async () => { @@ -41,7 +41,7 @@ describe('KTX embedding health check', () => { backend: 'openai', model: 'text-embedding-3-small', dimensions: 3, - openai: { apiKey: 'sk-openai-test' }, + openai: { apiKey: 'sk-openai-test' }, // pragma: allowlist secret }, { deps: { createOpenAIClient } }, ), @@ -66,7 +66,7 @@ describe('KTX embedding health check', () => { backend: 'openai', model: 'text-embedding-3-small', dimensions: 3, - openai: { apiKey: 'sk-openai-secret' }, + openai: { apiKey: 'sk-openai-secret' }, // pragma: allowlist secret }, { deps: { createOpenAIClient } }, ), @@ -94,7 +94,7 @@ describe('KTX embedding health check', () => { backend: 'openai', model: 'text-embedding-3-small', dimensions: 3, - openai: { apiKey: 'sk-openai-test' }, + openai: { apiKey: 'sk-openai-test' }, // pragma: allowlist secret }, { timeoutMs: 1, deps: { createOpenAIClient } }, ), diff --git a/packages/llm/src/model-health.test.ts b/packages/llm/src/model-health.test.ts index d1b3df47..8752b09e 100644 --- a/packages/llm/src/model-health.test.ts +++ b/packages/llm/src/model-health.test.ts @@ -14,7 +14,7 @@ describe('KTX LLM health check', () => { runKtxLlmHealthCheck( { backend: 'anthropic', - anthropic: { apiKey: 'sk-ant-test' }, + anthropic: { apiKey: 'sk-ant-test' }, // pragma: allowlist secret modelSlots: { default: 'claude-sonnet-4-6' }, }, { deps: { createAnthropic, generateText, devtoolsEnabled: true, wrapLanguageModel } }, @@ -23,7 +23,7 @@ describe('KTX LLM health check', () => { expect(createAnthropic).toHaveBeenCalledWith( expect.objectContaining({ - apiKey: 'sk-ant-test', + apiKey: 'sk-ant-test', // pragma: allowlist secret }), ); expect(generateText).toHaveBeenCalledWith( @@ -46,7 +46,7 @@ describe('KTX LLM health check', () => { runKtxLlmHealthCheck( { backend: 'anthropic', - anthropic: { apiKey: 'sk-ant-secret' }, + anthropic: { apiKey: 'sk-ant-secret' }, // pragma: allowlist secret modelSlots: { default: 'claude-sonnet-4-6' }, }, { diff --git a/pyproject.toml b/pyproject.toml index 1c4816b7..e6422fb6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ Issues = "https://github.com/kaelio/ktx/issues" [dependency-groups] dev = [ + "pre-commit>=4.6.0", "pytest>=9.0.2", "ruff>=0.8.4", ] diff --git a/python/ktx-daemon/src/ktx_daemon/sql_analysis.py b/python/ktx-daemon/src/ktx_daemon/sql_analysis.py index 9a222098..d5deb240 100644 --- a/python/ktx-daemon/src/ktx_daemon/sql_analysis.py +++ b/python/ktx-daemon/src/ktx_daemon/sql_analysis.py @@ -130,7 +130,9 @@ def _analyze_one( ) -def _analyze_payload(payload: tuple[str, str, str]) -> tuple[str, AnalyzeSqlBatchResult]: +def _analyze_payload( + payload: tuple[str, str, str], +) -> tuple[str, AnalyzeSqlBatchResult]: item_id, sql, dialect = payload return _analyze_one(item_id, sql, dialect) diff --git a/scripts/public-benchmark-manifest.json b/scripts/public-benchmark-manifest.json index e106e24e..fdd97e59 100644 --- a/scripts/public-benchmark-manifest.json +++ b/scripts/public-benchmark-manifest.json @@ -4,7 +4,7 @@ "id": "chinook_with_declared_metadata", "displayName": "Chinook (SQLite, declared metadata)", "url": "https://raw.githubusercontent.com/lerocha/chinook-database/master/ChinookDatabase/DataSources/Chinook_Sqlite.sqlite", - "sha256": "7651ba378ac2fcd0dfc3c66fb101f7a7eed3ba39a612ec642b96e20702061f15", + "sha256": "7651ba378ac2fcd0dfc3c66fb101f7a7eed3ba39a612ec642b96e20702061f15", "_allowlist": "// pragma: allowlist secret", "license": "MIT", "source": "https://github.com/lerocha/chinook-database" }, @@ -12,7 +12,7 @@ "id": "northwind_with_declared_metadata", "displayName": "Northwind (SQLite, declared metadata)", "url": "https://github.com/jpwhite3/northwind-SQLite3/raw/main/dist/northwind.db", - "sha256": "2f4f5c68dfcd33ba27373eae48c7a4869800c68095ee0f9f0da494f83382a877", + "sha256": "2f4f5c68dfcd33ba27373eae48c7a4869800c68095ee0f9f0da494f83382a877", "_allowlist": "// pragma: allowlist secret", "license": "MIT", "source": "https://github.com/jpwhite3/northwind-SQLite3" }, @@ -20,7 +20,7 @@ "id": "sakila_with_declared_metadata", "displayName": "Sakila (SQLite, declared metadata)", "url": "https://raw.githubusercontent.com/bradleygrant/sakila-sqlite3/master/sakila_master.db", - "sha256": "88c91a4a1a6b61f9d3f35904c0a173c887b25e73f20c3c2fdb073818c06f4268", + "sha256": "88c91a4a1a6b61f9d3f35904c0a173c887b25e73f20c3c2fdb073818c06f4268", "_allowlist": "// pragma: allowlist secret", "license": "BSD-2-Clause", "source": "https://github.com/bradleygrant/sakila-sqlite3" }, diff --git a/scripts/standalone-ci-workflow.test.mjs b/scripts/standalone-ci-workflow.test.mjs index 195fce53..5aa4cc02 100644 --- a/scripts/standalone-ci-workflow.test.mjs +++ b/scripts/standalone-ci-workflow.test.mjs @@ -20,6 +20,8 @@ describe('standalone KTX CI workflow', () => { assertIncludesAll(workflow, [ 'permissions:', 'contents: read', + 'pre-commit-checks:', + 'name: Pre-commit checks', 'typescript-checks:', 'name: TypeScript checks', 'slow-context-tests:', @@ -33,7 +35,7 @@ describe('standalone KTX CI workflow', () => { 'artifact-checks:', 'name: Artifact checks', 'actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd', - 'pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0', + 'pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093', 'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e', 'node-version: "24"', 'cache-dependency-path: "pnpm-lock.yaml"', @@ -46,7 +48,10 @@ describe('standalone KTX CI workflow', () => { 'actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405', 'python-version: "3.13"', 'astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b', + 'version: "0.11.11"', 'cache-dependency-glob: "uv.lock"', + 'uv sync --all-packages --all-groups', + 'uv run pre-commit run --all-files', 'uv sync --all-packages', 'uv run pytest', 'pnpm run artifacts:check', diff --git a/uv.lock b/uv.lock index 5458900e..5531c8e3 100644 --- a/uv.lock +++ b/uv.lock @@ -546,6 +546,7 @@ source = { virtual = "." } [package.dev-dependencies] dev = [ + { name = "pre-commit" }, { name = "pytest" }, { name = "ruff" }, ] @@ -554,6 +555,7 @@ dev = [ [package.metadata.requires-dev] dev = [ + { name = "pre-commit", specifier = ">=4.6.0" }, { name = "pytest", specifier = ">=9.0.2" }, { name = "ruff", specifier = ">=0.8.4" }, ]