From fa9237956eda616a457834e18c9efd89b060c27b Mon Sep 17 00:00:00 2001 From: Andrey Avtomonov Date: Wed, 13 May 2026 19:49:25 +0200 Subject: [PATCH 1/7] ci: run pre-commit checks in CI (#74) * ci: run pre-commit in CI * test: update CI workflow guardrail --- .github/workflows/ci.yml | 50 +++++++++++++++++-- .github/workflows/release.yml | 2 +- LICENSE | 1 - .../docs/integrations/primary-sources.mdx | 6 +-- ...11-historic-sql-cross-dialect-readiness.md | 32 ++++++------ ...ric-sql-end-to-end-retrieval-acceptance.md | 2 +- .../2026-05-11-historic-sql-foundations.md | 42 ++++++++-------- ...1-historic-sql-pattern-shard-smoke-docs.md | 8 +-- ...-historic-sql-pattern-workunit-sharding.md | 18 +++---- ...-05-11-historic-sql-redaction-hardening.md | 12 ++--- ...26-05-11-historic-sql-search-enrichment.md | 22 ++++---- ...-historic-sql-skills-projection-cutover.md | 50 +++++++++---------- packages/cli/src/connection.test.ts | 2 +- packages/cli/src/setup-embeddings.test.ts | 14 +++--- .../adapters/historic-sql/redaction.test.ts | 2 +- .../historic-sql/stage-unified.test.ts | 2 +- .../ingest/adapters/metabase/client.test.ts | 2 +- .../metabase/local-metabase.adapter.test.ts | 2 +- packages/llm/src/embedding-health.test.ts | 10 ++-- packages/llm/src/model-health.test.ts | 6 +-- pyproject.toml | 1 + .../ktx-daemon/src/ktx_daemon/sql_analysis.py | 4 +- scripts/public-benchmark-manifest.json | 6 +-- scripts/standalone-ci-workflow.test.mjs | 7 ++- uv.lock | 2 + 25 files changed, 177 insertions(+), 128 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3da14c7b..ff0b7843 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,6 +15,46 @@ concurrency: cancel-in-progress: true jobs: + pre-commit-checks: + name: Pre-commit checks + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Setup pnpm + uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 + with: + run_install: false + + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: "24" + cache: "pnpm" + cache-dependency-path: "pnpm-lock.yaml" + + - name: Install TypeScript dependencies + run: pnpm install --frozen-lockfile + + - name: Setup Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.13" + + - name: Setup uv + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + with: + version: "0.11.11" + enable-cache: true + cache-dependency-glob: "uv.lock" + + - name: Install Python dependencies + run: uv sync --all-packages --all-groups + + - name: Run pre-commit hooks + run: uv run pre-commit run --all-files + typescript-checks: name: TypeScript checks runs-on: ubuntu-latest @@ -23,7 +63,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup pnpm - uses: pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0 # v6.0.7 + uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 with: run_install: false @@ -51,7 +91,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup pnpm - uses: pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0 # v6.0.7 + uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 with: run_install: false @@ -79,7 +119,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup pnpm - uses: pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0 # v6.0.7 + uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 with: run_install: false @@ -107,7 +147,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup pnpm - uses: pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0 # v6.0.7 + uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 with: run_install: false @@ -156,7 +196,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup pnpm - uses: pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0 # v6.0.7 + uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 with: run_install: false diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2a8f696e..36eaf49c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,7 +24,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup pnpm - uses: pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0 # v6.0.7 + uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 with: run_install: false diff --git a/LICENSE b/LICENSE index 57bc88a1..261eeb9e 100644 --- a/LICENSE +++ b/LICENSE @@ -199,4 +199,3 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - diff --git a/docs-site/content/docs/integrations/primary-sources.mdx b/docs-site/content/docs/integrations/primary-sources.mdx index 653c4e38..f3abf9f0 100644 --- a/docs-site/content/docs/integrations/primary-sources.mdx +++ b/docs-site/content/docs/integrations/primary-sources.mdx @@ -34,7 +34,7 @@ The most full-featured connector. Supports schema introspection, foreign key det connections: my-postgres: driver: postgres - url: postgresql://user:password@host:5432/database + url: env:DATABASE_URL schema: public ``` @@ -320,7 +320,7 @@ Standard MySQL/MariaDB connector with full foreign key support and schema intros connections: my-mysql: driver: mysql - url: mysql://user:password@host:3306/database + url: env:MYSQL_DATABASE_URL ``` Or with individual fields: @@ -377,7 +377,7 @@ Connects to Microsoft SQL Server and Azure SQL. Supports multi-schema scanning w connections: my-sqlserver: driver: sqlserver - url: mssql://user:password@host:1433/database?trustServerCertificate=true + url: env:SQLSERVER_DATABASE_URL ``` Or with individual fields: diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-cross-dialect-readiness.md b/docs/superpowers/plans/2026-05-11-historic-sql-cross-dialect-readiness.md index a7a5cc6c..3fc3e496 100644 --- a/docs/superpowers/plans/2026-05-11-historic-sql-cross-dialect-readiness.md +++ b/docs/superpowers/plans/2026-05-11-historic-sql-cross-dialect-readiness.md @@ -40,37 +40,37 @@ This plan does not update `examples/postgres-historic/README.md` or `examples/po Modify: -- `packages/context/src/ingest/adapters/historic-sql/types.ts` +- `packages/context/src/ingest/adapters/historic-sql/types.ts` Adds optional probe `info` notes and lets injected historic-SQL dependencies use any reader/query client pair while preserving the existing Postgres-specific option. -- `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.ts` +- `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.ts` Moves low `pg_stat_statements.max` from `warnings` to `info`. -- `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.test.ts` Locks `track = none` as warning and low `max` as info. -- `packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.ts` +- `packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.ts` Returns `{ warnings: [], info: [] }` from `probe()`. -- `packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.test.ts` Locks the BigQuery probe return object. -- `packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.ts` +- `packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.ts` Returns `{ warnings: [], info: [] }` from `probe()`. -- `packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.test.ts` Locks the Snowflake probe return object. -- `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts` Updates test readers to return the normalized probe shape. -- `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts` Updates test readers to return the normalized probe shape. -- `packages/context/src/ingest/local-adapters.ts` +- `packages/context/src/ingest/local-adapters.ts` Accepts generic historic-SQL reader/query-client dependencies while keeping `postgresQueryClient` as the compatibility input used by current callers. -- `packages/context/src/ingest/local-adapters.test.ts` +- `packages/context/src/ingest/local-adapters.test.ts` Verifies generic reader/query-client injection and the existing Postgres compatibility path. -- `packages/cli/src/local-adapters.ts` +- `packages/cli/src/local-adapters.ts` Chooses Postgres, BigQuery, or Snowflake historic-SQL readers/query clients from the configured connection. -- `packages/cli/src/local-adapters.test.ts` +- `packages/cli/src/local-adapters.test.ts` Adds direct tests for CLI local adapter registration for Postgres, BigQuery, and Snowflake. -- `packages/cli/src/historic-sql-doctor.ts` +- `packages/cli/src/historic-sql-doctor.ts` Treats info-only Postgres probe notes as a passing doctor check, and warnings as warnings. -- `packages/cli/src/historic-sql-doctor.test.ts` +- `packages/cli/src/historic-sql-doctor.test.ts` Verifies low `pg_stat_statements.max` is pass/detail, while `track = none` remains warn. -- `packages/cli/src/doctor.test.ts` +- `packages/cli/src/doctor.test.ts` Updates the project doctor integration expectation for the new info-only behavior. ## Task 1: Normalize Historic-SQL Probe Results diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-end-to-end-retrieval-acceptance.md b/docs/superpowers/plans/2026-05-11-historic-sql-end-to-end-retrieval-acceptance.md index c9c40fd9..106131ed 100644 --- a/docs/superpowers/plans/2026-05-11-historic-sql-end-to-end-retrieval-acceptance.md +++ b/docs/superpowers/plans/2026-05-11-historic-sql-end-to-end-retrieval-acceptance.md @@ -44,7 +44,7 @@ Remaining acceptance gap this plan covers: Create: -- `packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts` Owns the end-to-end local regression for the redesigned historic-SQL pipeline. It uses the real adapter and local ingest runner, with fake deterministic reader/analysis/agent components so the test does not need a live database or LLM provider. ## Task 1: Add Real-Adapter Local Ingest Acceptance Coverage diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-foundations.md b/docs/superpowers/plans/2026-05-11-historic-sql-foundations.md index fdd97d3f..6705d56f 100644 --- a/docs/superpowers/plans/2026-05-11-historic-sql-foundations.md +++ b/docs/superpowers/plans/2026-05-11-historic-sql-foundations.md @@ -41,50 +41,50 @@ The next plan after this one should cover search enrichment from spec §6.2.3-§ Create: -- `packages/context/src/ingest/adapters/historic-sql/skill-schemas.ts` +- `packages/context/src/ingest/adapters/historic-sql/skill-schemas.ts` Owns the shared zod schemas for historic-SQL LLM outputs. -- `packages/context/src/ingest/adapters/historic-sql/skill-schemas.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/skill-schemas.test.ts` Locks schema acceptance, JSON schema generation, and future-key tolerance. -- `python/ktx-daemon/src/ktx_daemon/sql_analysis.py` +- `python/ktx-daemon/src/ktx_daemon/sql_analysis.py` Implements batch sqlglot parsing for table and clause-level column extraction. -- `python/ktx-daemon/tests/test_sql_analysis.py` +- `python/ktx-daemon/tests/test_sql_analysis.py` Tests batch parser behavior without FastAPI. Modify: -- `packages/context/src/ingest/index.ts` +- `packages/context/src/ingest/index.ts` Exports the new historic-SQL skill schemas. -- `packages/context/src/sl/types.ts` +- `packages/context/src/sl/types.ts` Adds `usage?: TableUsageOutput` to `SemanticLayerSource`. -- `packages/context/src/sl/schemas.ts` +- `packages/context/src/sl/schemas.ts` Accepts `usage` in standalone and overlay semantic-layer source validation. -- `packages/context/src/sl/semantic-layer.service.ts` +- `packages/context/src/sl/semantic-layer.service.ts` Projects manifest `usage` onto `SemanticLayerSource` and composes overlay usage intentionally. -- `packages/context/src/sl/semantic-layer.service.test.ts` +- `packages/context/src/sl/semantic-layer.service.test.ts` Tests source schema acceptance, manifest projection, and overlay composition. -- `packages/context/src/ingest/adapters/live-database/manifest.ts` +- `packages/context/src/ingest/adapters/live-database/manifest.ts` Adds `LiveDatabaseManifestTableEntry.usage`, existing-usage inputs, and `mergeUsagePreservingExternal()`. -- `packages/context/src/ingest/adapters/live-database/manifest.test.ts` +- `packages/context/src/ingest/adapters/live-database/manifest.test.ts` Tests scan-managed usage replacement while preserving external keys. -- `packages/context/src/scan/local-enrichment-artifacts.ts` +- `packages/context/src/scan/local-enrichment-artifacts.ts` Loads existing manifest usage and passes it through scan manifest rebuilds. -- `packages/context/src/scan/local-enrichment-artifacts.test.ts` +- `packages/context/src/scan/local-enrichment-artifacts.test.ts` Tests that structural scan rewrites preserve existing usage. -- `python/ktx-daemon/src/ktx_daemon/app.py` +- `python/ktx-daemon/src/ktx_daemon/app.py` Registers `/sql/analyze-batch`. -- `python/ktx-daemon/tests/test_app.py` +- `python/ktx-daemon/tests/test_app.py` Tests the FastAPI endpoint. -- `packages/context/src/sql-analysis/ports.ts` +- `packages/context/src/sql-analysis/ports.ts` Adds batch analysis types and `SqlAnalysisPort.analyzeBatch()`. -- `packages/context/src/sql-analysis/index.ts` +- `packages/context/src/sql-analysis/index.ts` Exports the new batch analysis types. -- `packages/context/src/sql-analysis/http-sql-analysis-port.ts` +- `packages/context/src/sql-analysis/http-sql-analysis-port.ts` Maps `/sql/analyze-batch` request and response payloads. -- `packages/context/src/sql-analysis/http-sql-analysis-port.test.ts` +- `packages/context/src/sql-analysis/http-sql-analysis-port.test.ts` Tests HTTP mapping and malformed response rejection. -- `packages/cli/src/managed-python-http.test.ts` +- `packages/cli/src/managed-python-http.test.ts` Verifies the managed daemon wrapper routes `analyzeBatch()`. -- Existing test files with `SqlAnalysisPort` object literals +- Existing test files with `SqlAnalysisPort` object literals Add a no-op `analyzeBatch: async () => new Map()` while legacy paths still use `analyzeForFingerprint()`. ## Task 1: Add Historic SQL Skill Schemas diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-pattern-shard-smoke-docs.md b/docs/superpowers/plans/2026-05-11-historic-sql-pattern-shard-smoke-docs.md index 9e386a16..b5382ff4 100644 --- a/docs/superpowers/plans/2026-05-11-historic-sql-pattern-shard-smoke-docs.md +++ b/docs/superpowers/plans/2026-05-11-historic-sql-pattern-shard-smoke-docs.md @@ -39,13 +39,13 @@ Remaining gap this plan fixes: ## File Structure -- Modify `scripts/examples-docs.test.mjs` +- Modify `scripts/examples-docs.test.mjs` Pins docs and smoke script to the sharded pattern WorkUnit contract. -- Modify `examples/postgres-historic/scripts/smoke.sh` +- Modify `examples/postgres-historic/scripts/smoke.sh` Validates `patterns-input/part-*.json` shard files and `historic-sql-patterns-part-*` stage-only WorkUnits. -- Modify `examples/postgres-historic/README.md` +- Modify `examples/postgres-historic/README.md` Documents `patterns-input.json` as the full audit artifact and `patterns-input/part-*.json` as bounded pattern WorkUnit input. -- Modify `examples/README.md` +- Modify `examples/README.md` Updates the short example catalog entry with the same audit-vs-shard wording. ### Task 1: Pin Example Tests To Pattern Shards diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-pattern-workunit-sharding.md b/docs/superpowers/plans/2026-05-11-historic-sql-pattern-workunit-sharding.md index ee7604a7..c67f6d78 100644 --- a/docs/superpowers/plans/2026-05-11-historic-sql-pattern-workunit-sharding.md +++ b/docs/superpowers/plans/2026-05-11-historic-sql-pattern-workunit-sharding.md @@ -30,23 +30,23 @@ No existing spec-derived plan is currently unimplemented in this worktree. This ## File Structure -- Create `packages/context/src/ingest/adapters/historic-sql/pattern-inputs.ts` +- Create `packages/context/src/ingest/adapters/historic-sql/pattern-inputs.ts` Owns deterministic pattern audit ordering, cross-table candidate filtering, byte-bounded shard creation, shard path constants, and shard path detection. -- Create `packages/context/src/ingest/adapters/historic-sql/pattern-inputs.test.ts` +- Create `packages/context/src/ingest/adapters/historic-sql/pattern-inputs.test.ts` Covers deterministic shard ordering, single-table exclusion from WorkUnit shards, byte limits, and oversize-template manifest warnings. -- Modify `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts` +- Modify `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts` Writes full `patterns-input.json` plus bounded `patterns-input/part-0001.json` shard files, and appends shard warnings to `manifest.json`. -- Modify `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts` +- Modify `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts` Adds a regression for audit file preservation and sharded WorkUnit input creation. -- Modify `packages/context/src/ingest/adapters/historic-sql/chunk-unified.ts` +- Modify `packages/context/src/ingest/adapters/historic-sql/chunk-unified.ts` Emits one patterns WorkUnit per changed shard path, treats root `patterns-input.json` as audit-only, and includes shard paths in the scope descriptor and eviction calculation. -- Modify `packages/context/src/ingest/adapters/historic-sql/chunk-unified.test.ts` +- Modify `packages/context/src/ingest/adapters/historic-sql/chunk-unified.test.ts` Updates root-file expectations and adds multi-shard diff behavior. -- Modify `packages/context/skills/historic_sql_patterns/SKILL.md` +- Modify `packages/context/skills/historic_sql_patterns/SKILL.md` Tells the skill to read the exact pattern shard in `rawFiles` and emit evidence with that shard as `rawPath`. -- Modify `packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts` +- Modify `packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts` Updates the fake agent to emit pattern evidence for `historic-sql-patterns-part-0001`. -- Modify `packages/context/src/ingest/ingest-runtime-assets.test.ts` +- Modify `packages/context/src/ingest/ingest-runtime-assets.test.ts` Keeps packaged skill assertions aligned with sharded pattern file guidance. ## Task 1: Add Pattern Input Sharding Helper diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-redaction-hardening.md b/docs/superpowers/plans/2026-05-11-historic-sql-redaction-hardening.md index 1adcdfd3..e59e164b 100644 --- a/docs/superpowers/plans/2026-05-11-historic-sql-redaction-hardening.md +++ b/docs/superpowers/plans/2026-05-11-historic-sql-redaction-hardening.md @@ -55,16 +55,16 @@ Remaining spec gap this plan covers: Create: -- `packages/context/src/ingest/adapters/historic-sql/redaction.ts` +- `packages/context/src/ingest/adapters/historic-sql/redaction.ts` Owns compilation and application of historic-SQL SQL-text redaction patterns. Supports JavaScript regex strings and the documented `(?i)` case-insensitive prefix used by setup tests/docs. -- `packages/context/src/ingest/adapters/historic-sql/redaction.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/redaction.test.ts` Tests raw regex replacement, `(?i)` compatibility, empty config behavior, and invalid-pattern diagnostics. Modify: -- `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts` +- `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts` Compiles `config.redactionPatterns` once per fetch. Keeps original SQL for filtering and `SqlAnalysisPort.analyzeBatch()`, then stores redacted SQL in `ParsedTemplate.template.canonicalSql` before `toStagedTable()` and `toPatternsInput()` serialize files. -- `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts` Adds a regression proving raw secrets are absent from staged artifacts while `analyzeBatch()` still receives the original SQL. ## Task 1: Add Historic SQL Redaction Helper @@ -89,7 +89,7 @@ describe('historic-SQL redaction', () => { ]); const sql = - "select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; + "select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; // pragma: allowlist secret expect(redactHistoricSqlText(sql, redactors)).toBe( "select * from public.api_events where api_key = '[REDACTED]' and note = '[REDACTED]'", @@ -202,7 +202,7 @@ Append this test inside the existing `describe('stageHistoricSqlAggregatedSnapsh it('redacts configured SQL substrings in staged artifacts while analyzing original SQL', async () => { const stagedDir = await tempDir(); const originalSql = - "select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; + "select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; // pragma: allowlist secret const reader: HistoricSqlReader = { async probe() { return { warnings: [], info: [] }; diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-search-enrichment.md b/docs/superpowers/plans/2026-05-11-historic-sql-search-enrichment.md index ee960bb8..cafc234b 100644 --- a/docs/superpowers/plans/2026-05-11-historic-sql-search-enrichment.md +++ b/docs/superpowers/plans/2026-05-11-historic-sql-search-enrichment.md @@ -37,27 +37,27 @@ This plan does not rewrite the historic-SQL adapter, readers, skills, projection Modify: -- `packages/context/src/sl/sl-search.service.ts` +- `packages/context/src/sl/sl-search.service.ts` Adds usage narrative, frequency, filters, group-bys, joins, and stale marker to the canonical SL search text. Preserves snippets returned by repository search for direct `SlSearchService.search()` callers. -- `packages/context/src/sl/sl-search.service.test.ts` +- `packages/context/src/sl/sl-search.service.test.ts` Tests usage search-text content and direct service snippet pass-through. -- `packages/context/src/sl/ports.ts` +- `packages/context/src/sl/ports.ts` Extends `SlSourcesIndexPort.search()` rows with optional `snippet`. -- `packages/context/src/sl/sqlite-sl-sources-index.ts` +- `packages/context/src/sl/sqlite-sl-sources-index.ts` Adds FTS5 `snippet()` selection to lexical candidate search and direct index search. -- `packages/context/src/sl/sqlite-sl-sources-index.test.ts` +- `packages/context/src/sl/sqlite-sl-sources-index.test.ts` Locks snippet behavior for both direct search and lexical lane candidates. -- `packages/context/src/sl/local-sl.ts` +- `packages/context/src/sl/local-sl.ts` Adds `frequencyTier` and `snippet` to query-mode `LocalSlSourceSearchResult`; collects snippets from the lexical lane and hydrates frequency from `SemanticLayerSource.usage`. -- `packages/context/src/sl/local-sl.test.ts` +- `packages/context/src/sl/local-sl.test.ts` Tests that usage-only terms can find a source and that results include `frequencyTier` and FTS snippet. -- `packages/context/src/sl/pglite-sl-search-prototype.ts` +- `packages/context/src/sl/pglite-sl-search-prototype.ts` Propagates `frequencyTier` for the prototype backend so the shared result type stays truthful. -- `packages/context/src/mcp/types.ts` +- `packages/context/src/mcp/types.ts` Adds `frequencyTier` and `snippet` to `KtxSemanticLayerSourceSummary`. -- `packages/context/src/mcp/local-project-ports.ts` +- `packages/context/src/mcp/local-project-ports.ts` Includes `frequencyTier` and `snippet` in `semanticLayer.listSources()` output. -- `packages/context/src/mcp/local-project-ports.test.ts` +- `packages/context/src/mcp/local-project-ports.test.ts` Tests the agent/MCP-facing list response. ## Task 1: Index Historic SQL Usage In SL Search Text diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-skills-projection-cutover.md b/docs/superpowers/plans/2026-05-11-historic-sql-skills-projection-cutover.md index a892542e..a7494e2d 100644 --- a/docs/superpowers/plans/2026-05-11-historic-sql-skills-projection-cutover.md +++ b/docs/superpowers/plans/2026-05-11-historic-sql-skills-projection-cutover.md @@ -52,58 +52,58 @@ Still not implemented: Create: -- `packages/context/src/ingest/adapters/historic-sql/evidence.ts` +- `packages/context/src/ingest/adapters/historic-sql/evidence.ts` Owns typed evidence envelopes, ignored evidence path helpers, and load/write helpers for table usage and pattern evidence. -- `packages/context/src/ingest/adapters/historic-sql/evidence.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/evidence.test.ts` Tests evidence schema validation, path normalization, and loader rejection of malformed evidence. -- `packages/context/src/ingest/adapters/historic-sql/evidence-tool.ts` +- `packages/context/src/ingest/adapters/historic-sql/evidence-tool.ts` Adds `emit_historic_sql_evidence`, the only write tool the two new historic-SQL skills use. -- `packages/context/src/ingest/adapters/historic-sql/evidence-tool.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/evidence-tool.test.ts` Tests the tool writes ignored run-local JSON with `skipLock: true` and rejects non-historic ingest sessions. -- `packages/context/src/ingest/adapters/historic-sql/projection.ts` +- `packages/context/src/ingest/adapters/historic-sql/projection.ts` Projects table usage evidence into manifest shards, writes pattern wiki pages, marks stale usage/pages, and deletes legacy query pages. -- `packages/context/src/ingest/adapters/historic-sql/projection.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/projection.test.ts` Tests `_schema` merge, stale usage, pattern slug reuse, stale page tagging, archive movement, and legacy page cleanup. -- `packages/context/src/ingest/adapters/historic-sql/post-processor.ts` +- `packages/context/src/ingest/adapters/historic-sql/post-processor.ts` Implements `IngestBundlePostProcessorPort` for the deterministic projection phase. -- `packages/context/src/ingest/adapters/historic-sql/post-processor.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/post-processor.test.ts` Tests post-processor path resolution from `workdir`, `connectionId`, `sourceKey`, and `syncId`. -- `packages/context/skills/historic_sql_table_digest/SKILL.md` +- `packages/context/skills/historic_sql_table_digest/SKILL.md` Skill for one changed `tables/*.json` WorkUnit; emits one table usage evidence object. -- `packages/context/skills/historic_sql_patterns/SKILL.md` +- `packages/context/skills/historic_sql_patterns/SKILL.md` Skill for `patterns-input.json`; emits one pattern evidence object per recurring cross-table intent. Modify: -- `packages/context/src/ingest/adapters/historic-sql/types.ts` +- `packages/context/src/ingest/adapters/historic-sql/types.ts` Keep only unified config/staged schemas and reader contracts; extend config preprocessing for existing `serviceAccountUserPatterns` and `minCalls` aliases. -- `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts` +- `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts` Add `staleArchiveAfterDays` to `manifest.json` so projection can archive stale pattern pages deterministically. -- `packages/context/src/ingest/adapters/historic-sql/chunk-unified.ts` +- `packages/context/src/ingest/adapters/historic-sql/chunk-unified.ts` Keep the same WorkUnits, but mention `emit_historic_sql_evidence` in `notes`. -- `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.ts` +- `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.ts` Switch production fetch/chunk/scope to the unified hot path, replace skills, remove legacy triage support, and run legacy PGSS baseline cache cleanup. -- `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts` Rewrite around unified staging and new skills. -- `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.ts` +- `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.ts` Inline the PGSS probe logic so `postgres-pgss-query-history-reader.ts` can be deleted. -- `packages/context/src/ingest/local-adapters.ts` +- `packages/context/src/ingest/local-adapters.ts` Use `PostgresPgssReader` for local Postgres historic SQL and return unified pull config. -- `packages/context/src/ingest/local-bundle-runtime.ts` +- `packages/context/src/ingest/local-bundle-runtime.ts` Add the source-specific evidence tool to historic-SQL WorkUnits and register the historic-SQL post-processor. -- `packages/context/src/ingest/ingest-runtime-assets.test.ts` +- `packages/context/src/ingest/ingest-runtime-assets.test.ts` Replace old skill asset assertions with the two new skills. -- `packages/context/src/memory/memory-runtime-assets.test.ts` +- `packages/context/src/memory/memory-runtime-assets.test.ts` Replace old historic-SQL skill heading with the two new skill headings. -- `packages/context/src/package-exports.test.ts` +- `packages/context/src/package-exports.test.ts` Remove legacy export assertions and add evidence/projection export assertions. -- `packages/context/src/ingest/index.ts` +- `packages/context/src/ingest/index.ts` Export new evidence/projection/post-processor helpers and remove legacy historic-SQL exports. -- `packages/cli/src/setup-databases.ts` and `packages/cli/src/historic-sql-doctor.ts` +- `packages/cli/src/setup-databases.ts` and `packages/cli/src/historic-sql-doctor.ts` Import `PostgresPgssReader` instead of `PostgresPgssQueryHistoryReader`. -- `packages/cli/src/commands/setup-commands.ts`, `packages/cli/src/index.test.ts`, `packages/cli/src/setup-databases.test.ts` +- `packages/cli/src/commands/setup-commands.ts`, `packages/cli/src/index.test.ts`, `packages/cli/src/setup-databases.test.ts` Rename generated config to `minExecutions` while accepting the old `--historic-sql-min-calls` flag for one release. -- `packages/context/prompts/skills/page_triage_classifier.md`, `packages/context/src/ingest/page-triage/page-triage.service.test.ts`, `packages/context/src/ingest/ingest-prompts.test.ts` +- `packages/context/prompts/skills/page_triage_classifier.md`, `packages/context/src/ingest/page-triage/page-triage.service.test.ts`, `packages/context/src/ingest/ingest-prompts.test.ts` Remove historic-SQL template triage examples because the new adapter no longer uses page triage. Delete: diff --git a/packages/cli/src/connection.test.ts b/packages/cli/src/connection.test.ts index 6eb3a08c..0d592b00 100644 --- a/packages/cli/src/connection.test.ts +++ b/packages/cli/src/connection.test.ts @@ -159,7 +159,7 @@ describe('runKtxConnection', () => { prod_metabase: { driver: 'metabase', api_url: 'http://metabase.example.test', - api_key: 'mb_test', + api_key: 'mb_test', // pragma: allowlist secret }, }); const testConnection = vi.fn(async () => ({ success: true as const })); diff --git a/packages/cli/src/setup-embeddings.test.ts b/packages/cli/src/setup-embeddings.test.ts index c2d5cad2..4c91ddd5 100644 --- a/packages/cli/src/setup-embeddings.test.ts +++ b/packages/cli/src/setup-embeddings.test.ts @@ -319,14 +319,14 @@ describe('setup embeddings step', () => { projectDir: tempDir, inputMode: 'disabled', embeddingBackend: 'openai', - embeddingApiKeyEnv: 'OPENAI_API_KEY', + embeddingApiKeyEnv: 'OPENAI_API_KEY', // pragma: allowlist secret cliVersion: '0.2.0', runtimeInstallPolicy: 'auto', skipEmbeddings: false, }, io.io, { - env: { OPENAI_API_KEY: 'sk-openai-test' }, + env: { OPENAI_API_KEY: 'sk-openai-test' }, // pragma: allowlist secret healthCheck, }, ); @@ -336,14 +336,14 @@ describe('setup embeddings step', () => { backend: 'openai', model: 'text-embedding-3-small', dimensions: 1536, - openai: { apiKey: 'sk-openai-test' }, + openai: { apiKey: 'sk-openai-test' }, // pragma: allowlist secret }); const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); expect(config.ingest.embeddings).toMatchObject({ backend: 'openai', model: 'text-embedding-3-small', dimensions: 1536, - openai: { api_key: 'env:OPENAI_API_KEY' }, + openai: { api_key: 'env:OPENAI_API_KEY' }, // pragma: allowlist secret }); expect(io.stdout()).not.toContain('sk-openai-test'); }); @@ -367,7 +367,7 @@ describe('setup embeddings step', () => { io.io, { prompts, - env: { OPENAI_API_KEY: 'sk-openai-test' }, + env: { OPENAI_API_KEY: 'sk-openai-test' }, // pragma: allowlist secret healthCheck, ensureLocalEmbeddings: vi.fn(async () => managedDaemon()), }, @@ -384,7 +384,7 @@ describe('setup embeddings step', () => { backend: 'openai', model: 'text-embedding-3-small', dimensions: 1536, - openai: { apiKey: 'sk-openai-test' }, + openai: { apiKey: 'sk-openai-test' }, // pragma: allowlist secret }); expect(prompts.select).toHaveBeenCalledWith( expect.objectContaining({ @@ -478,7 +478,7 @@ describe('setup embeddings step', () => { }, makeIo().io, { - env: { OPENAI_API_KEY: 'sk-openai-test' }, + env: { OPENAI_API_KEY: 'sk-openai-test' }, // pragma: allowlist secret healthCheck, }, ), diff --git a/packages/context/src/ingest/adapters/historic-sql/redaction.test.ts b/packages/context/src/ingest/adapters/historic-sql/redaction.test.ts index c8f1d78b..d27015a6 100644 --- a/packages/context/src/ingest/adapters/historic-sql/redaction.test.ts +++ b/packages/context/src/ingest/adapters/historic-sql/redaction.test.ts @@ -9,7 +9,7 @@ describe('historic-SQL redaction', () => { ]); const sql = - "select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; + "select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; // pragma: allowlist secret expect(redactHistoricSqlText(sql, redactors)).toBe( "select * from public.api_events where api_key = '[REDACTED]' and note = '[REDACTED]'", diff --git a/packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts b/packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts index 421970bf..d1610054 100644 --- a/packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts +++ b/packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts @@ -169,7 +169,7 @@ describe('stageHistoricSqlAggregatedSnapshot', () => { it('redacts configured SQL substrings in staged artifacts while analyzing original SQL', async () => { const stagedDir = await tempDir(); const originalSql = - "select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; + "select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; // pragma: allowlist secret const reader: HistoricSqlReader = { async probe() { return { warnings: [], info: [] }; diff --git a/packages/context/src/ingest/adapters/metabase/client.test.ts b/packages/context/src/ingest/adapters/metabase/client.test.ts index 1c0fdfa9..3d45a276 100644 --- a/packages/context/src/ingest/adapters/metabase/client.test.ts +++ b/packages/context/src/ingest/adapters/metabase/client.test.ts @@ -92,7 +92,7 @@ describe('MetabaseClient retry exhaustion', () => { .mockResolvedValueOnce(new Response(JSON.stringify([]), { status: 200 })); const client = new MetabaseClient( - { apiUrl: 'https://metabase.example.test', apiKey: 'key' }, + { apiUrl: 'https://metabase.example.test', apiKey: 'key' }, // pragma: allowlist secret { ...DEFAULT_METABASE_CLIENT_CONFIG, baseDelayMs: 0, diff --git a/packages/context/src/ingest/adapters/metabase/local-metabase.adapter.test.ts b/packages/context/src/ingest/adapters/metabase/local-metabase.adapter.test.ts index 7cbe913b..b25ea18b 100644 --- a/packages/context/src/ingest/adapters/metabase/local-metabase.adapter.test.ts +++ b/packages/context/src/ingest/adapters/metabase/local-metabase.adapter.test.ts @@ -39,7 +39,7 @@ describe('metabaseRuntimeConfigFromLocalConnection', () => { const connection: KtxProjectConnectionConfig = { driver: 'metabase', api_url: 'https://metabase.example.com', - api_key_ref: `file:${keyPath}`, + api_key_ref: `file:${keyPath}`, // pragma: allowlist secret }; expect(metabaseRuntimeConfigFromLocalConnection('prod-metabase', connection)).toEqual({ diff --git a/packages/llm/src/embedding-health.test.ts b/packages/llm/src/embedding-health.test.ts index ca998aa9..65956311 100644 --- a/packages/llm/src/embedding-health.test.ts +++ b/packages/llm/src/embedding-health.test.ts @@ -17,13 +17,13 @@ describe('KTX embedding health check', () => { backend: 'openai', model: 'text-embedding-3-small', dimensions: 3, - openai: { apiKey: 'sk-openai-test' }, + openai: { apiKey: 'sk-openai-test' }, // pragma: allowlist secret }, { deps: { createOpenAIClient } }, ), ).resolves.toEqual({ ok: true }); - expect(createOpenAIClient).toHaveBeenCalledWith({ apiKey: 'sk-openai-test', baseURL: undefined }); + expect(createOpenAIClient).toHaveBeenCalledWith({ apiKey: 'sk-openai-test', baseURL: undefined }); // pragma: allowlist secret }); it('returns failed when the provider returns the wrong dimensions', async () => { @@ -41,7 +41,7 @@ describe('KTX embedding health check', () => { backend: 'openai', model: 'text-embedding-3-small', dimensions: 3, - openai: { apiKey: 'sk-openai-test' }, + openai: { apiKey: 'sk-openai-test' }, // pragma: allowlist secret }, { deps: { createOpenAIClient } }, ), @@ -66,7 +66,7 @@ describe('KTX embedding health check', () => { backend: 'openai', model: 'text-embedding-3-small', dimensions: 3, - openai: { apiKey: 'sk-openai-secret' }, + openai: { apiKey: 'sk-openai-secret' }, // pragma: allowlist secret }, { deps: { createOpenAIClient } }, ), @@ -94,7 +94,7 @@ describe('KTX embedding health check', () => { backend: 'openai', model: 'text-embedding-3-small', dimensions: 3, - openai: { apiKey: 'sk-openai-test' }, + openai: { apiKey: 'sk-openai-test' }, // pragma: allowlist secret }, { timeoutMs: 1, deps: { createOpenAIClient } }, ), diff --git a/packages/llm/src/model-health.test.ts b/packages/llm/src/model-health.test.ts index d1b3df47..8752b09e 100644 --- a/packages/llm/src/model-health.test.ts +++ b/packages/llm/src/model-health.test.ts @@ -14,7 +14,7 @@ describe('KTX LLM health check', () => { runKtxLlmHealthCheck( { backend: 'anthropic', - anthropic: { apiKey: 'sk-ant-test' }, + anthropic: { apiKey: 'sk-ant-test' }, // pragma: allowlist secret modelSlots: { default: 'claude-sonnet-4-6' }, }, { deps: { createAnthropic, generateText, devtoolsEnabled: true, wrapLanguageModel } }, @@ -23,7 +23,7 @@ describe('KTX LLM health check', () => { expect(createAnthropic).toHaveBeenCalledWith( expect.objectContaining({ - apiKey: 'sk-ant-test', + apiKey: 'sk-ant-test', // pragma: allowlist secret }), ); expect(generateText).toHaveBeenCalledWith( @@ -46,7 +46,7 @@ describe('KTX LLM health check', () => { runKtxLlmHealthCheck( { backend: 'anthropic', - anthropic: { apiKey: 'sk-ant-secret' }, + anthropic: { apiKey: 'sk-ant-secret' }, // pragma: allowlist secret modelSlots: { default: 'claude-sonnet-4-6' }, }, { diff --git a/pyproject.toml b/pyproject.toml index 1c4816b7..e6422fb6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ Issues = "https://github.com/kaelio/ktx/issues" [dependency-groups] dev = [ + "pre-commit>=4.6.0", "pytest>=9.0.2", "ruff>=0.8.4", ] diff --git a/python/ktx-daemon/src/ktx_daemon/sql_analysis.py b/python/ktx-daemon/src/ktx_daemon/sql_analysis.py index 9a222098..d5deb240 100644 --- a/python/ktx-daemon/src/ktx_daemon/sql_analysis.py +++ b/python/ktx-daemon/src/ktx_daemon/sql_analysis.py @@ -130,7 +130,9 @@ def _analyze_one( ) -def _analyze_payload(payload: tuple[str, str, str]) -> tuple[str, AnalyzeSqlBatchResult]: +def _analyze_payload( + payload: tuple[str, str, str], +) -> tuple[str, AnalyzeSqlBatchResult]: item_id, sql, dialect = payload return _analyze_one(item_id, sql, dialect) diff --git a/scripts/public-benchmark-manifest.json b/scripts/public-benchmark-manifest.json index e106e24e..fdd97e59 100644 --- a/scripts/public-benchmark-manifest.json +++ b/scripts/public-benchmark-manifest.json @@ -4,7 +4,7 @@ "id": "chinook_with_declared_metadata", "displayName": "Chinook (SQLite, declared metadata)", "url": "https://raw.githubusercontent.com/lerocha/chinook-database/master/ChinookDatabase/DataSources/Chinook_Sqlite.sqlite", - "sha256": "7651ba378ac2fcd0dfc3c66fb101f7a7eed3ba39a612ec642b96e20702061f15", + "sha256": "7651ba378ac2fcd0dfc3c66fb101f7a7eed3ba39a612ec642b96e20702061f15", "_allowlist": "// pragma: allowlist secret", "license": "MIT", "source": "https://github.com/lerocha/chinook-database" }, @@ -12,7 +12,7 @@ "id": "northwind_with_declared_metadata", "displayName": "Northwind (SQLite, declared metadata)", "url": "https://github.com/jpwhite3/northwind-SQLite3/raw/main/dist/northwind.db", - "sha256": "2f4f5c68dfcd33ba27373eae48c7a4869800c68095ee0f9f0da494f83382a877", + "sha256": "2f4f5c68dfcd33ba27373eae48c7a4869800c68095ee0f9f0da494f83382a877", "_allowlist": "// pragma: allowlist secret", "license": "MIT", "source": "https://github.com/jpwhite3/northwind-SQLite3" }, @@ -20,7 +20,7 @@ "id": "sakila_with_declared_metadata", "displayName": "Sakila (SQLite, declared metadata)", "url": "https://raw.githubusercontent.com/bradleygrant/sakila-sqlite3/master/sakila_master.db", - "sha256": "88c91a4a1a6b61f9d3f35904c0a173c887b25e73f20c3c2fdb073818c06f4268", + "sha256": "88c91a4a1a6b61f9d3f35904c0a173c887b25e73f20c3c2fdb073818c06f4268", "_allowlist": "// pragma: allowlist secret", "license": "BSD-2-Clause", "source": "https://github.com/bradleygrant/sakila-sqlite3" }, diff --git a/scripts/standalone-ci-workflow.test.mjs b/scripts/standalone-ci-workflow.test.mjs index 195fce53..5aa4cc02 100644 --- a/scripts/standalone-ci-workflow.test.mjs +++ b/scripts/standalone-ci-workflow.test.mjs @@ -20,6 +20,8 @@ describe('standalone KTX CI workflow', () => { assertIncludesAll(workflow, [ 'permissions:', 'contents: read', + 'pre-commit-checks:', + 'name: Pre-commit checks', 'typescript-checks:', 'name: TypeScript checks', 'slow-context-tests:', @@ -33,7 +35,7 @@ describe('standalone KTX CI workflow', () => { 'artifact-checks:', 'name: Artifact checks', 'actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd', - 'pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0', + 'pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093', 'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e', 'node-version: "24"', 'cache-dependency-path: "pnpm-lock.yaml"', @@ -46,7 +48,10 @@ describe('standalone KTX CI workflow', () => { 'actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405', 'python-version: "3.13"', 'astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b', + 'version: "0.11.11"', 'cache-dependency-glob: "uv.lock"', + 'uv sync --all-packages --all-groups', + 'uv run pre-commit run --all-files', 'uv sync --all-packages', 'uv run pytest', 'pnpm run artifacts:check', diff --git a/uv.lock b/uv.lock index 5458900e..5531c8e3 100644 --- a/uv.lock +++ b/uv.lock @@ -546,6 +546,7 @@ source = { virtual = "." } [package.dev-dependencies] dev = [ + { name = "pre-commit" }, { name = "pytest" }, { name = "ruff" }, ] @@ -554,6 +555,7 @@ dev = [ [package.metadata.requires-dev] dev = [ + { name = "pre-commit", specifier = ">=4.6.0" }, { name = "pytest", specifier = ">=9.0.2" }, { name = "ruff", specifier = ">=0.8.4" }, ] From ea33e51e8f56ac426fae9383bebec2f97962fd10 Mon Sep 17 00:00:00 2001 From: Luca Martial Date: Wed, 13 May 2026 13:29:50 -0700 Subject: [PATCH 2/7] refactor(cli): remove interactive gcloud auth from Vertex AI setup Instead of spawning an interactive gcloud login flow, tell users to run gcloud auth application-default login themselves before continuing. Also adds a Vertex-specific model list and spinner progress for LLM health checks. Co-Authored-By: Claude Opus 4.6 (1M context) --- packages/cli/src/setup-models.test.ts | 87 +++++++------ packages/cli/src/setup-models.ts | 176 ++++++++++++-------------- 2 files changed, 129 insertions(+), 134 deletions(-) diff --git a/packages/cli/src/setup-models.test.ts b/packages/cli/src/setup-models.test.ts index e310ea90..e4425d69 100644 --- a/packages/cli/src/setup-models.test.ts +++ b/packages/cli/src/setup-models.test.ts @@ -7,10 +7,8 @@ import { BUNDLED_ANTHROPIC_MODELS, fetchAnthropicModels, type KtxSetupModelPromptAdapter, - runKtxSetupGcloudApplicationDefaultAuth, runKtxSetupAnthropicModelStep, } from './setup-models.js'; -import type { KtxCliIo } from './cli-runtime.js'; function makeIo() { let stdout = ''; @@ -34,6 +32,17 @@ function makeIo() { }; } +function makeSpinnerEvents() { + const events: string[] = []; + const spinner = vi.fn(() => ({ + start: (msg: string) => events.push(`start:${msg}`), + message: (msg: string) => events.push(`message:${msg}`), + stop: (msg: string) => events.push(`stop:${msg}`), + error: (msg: string) => events.push(`error:${msg}`), + })); + return { events, spinner }; +} + function makePromptAdapter(options: { providerChoice?: string; selectValues?: string[]; @@ -191,6 +200,7 @@ describe('setup Anthropic model step', () => { it('configures env credentials, selected model, prompt caching, and llm completion state', async () => { const io = makeIo(); + const { events: spinnerEvents, spinner } = makeSpinnerEvents(); const result = await runKtxSetupAnthropicModelStep( { projectDir: tempDir, @@ -203,6 +213,7 @@ describe('setup Anthropic model step', () => { { env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, // pragma: allowlist secret healthCheck: vi.fn(async () => ({ ok: true as const })), + spinner, }, ); @@ -219,6 +230,10 @@ describe('setup Anthropic model step', () => { expect(config.scan.enrichment.mode).toBe('llm'); expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:'); expect((await readKtxSetupState(tempDir)).completed_steps).toContain('llm'); + expect(spinnerEvents).toEqual([ + 'start:Checking Anthropic API LLM (claude-sonnet-4-6).', + 'stop:LLM test passed (Anthropic API, claude-sonnet-4-6)', + ]); expect(io.stdout()).toContain('LLM ready: yes'); expect(io.stdout()).not.toContain('sk-ant-test'); }); @@ -226,6 +241,7 @@ describe('setup Anthropic model step', () => { it('configures Vertex AI provider, selected model, prompt caching, and llm completion state', async () => { const io = makeIo(); const healthCheck = vi.fn(async () => ({ ok: true as const })); + const { events: spinnerEvents, spinner } = makeSpinnerEvents(); const result = await runKtxSetupAnthropicModelStep( { @@ -238,7 +254,7 @@ describe('setup Anthropic model step', () => { skipLlm: false, }, io.io, - { env: {}, healthCheck }, + { env: {}, healthCheck, spinner }, ); expect(result.status).toBe('ready'); @@ -260,13 +276,16 @@ describe('setup Anthropic model step', () => { expect(config.scan.enrichment.mode).toBe('llm'); expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:'); expect((await readKtxSetupState(tempDir)).completed_steps).toContain('llm'); + expect(spinnerEvents).toEqual([ + 'start:Checking Vertex AI LLM (claude-sonnet-4-6).', + 'stop:LLM test passed (Vertex AI, claude-sonnet-4-6)', + ]); expect(io.stdout()).toContain('LLM ready: yes (claude-sonnet-4-6)'); }); - it('can run gcloud auth for Vertex AI and infer project and default location', async () => { + it('uses existing Vertex AI credentials without offering to run gcloud auth', async () => { const io = makeIo(); - const prompts = makePromptAdapter({ selectValues: ['vertex', 'gcloud', 'local-gcp-project', 'claude-sonnet-4-6'] }); - const runGcloudAuth = vi.fn(async () => ({ ok: true as const })); + const prompts = makePromptAdapter({ selectValues: ['vertex', 'existing', 'local-gcp-project', 'claude-sonnet-4-6'] }); const readGcloudProject = vi.fn(async () => 'local-gcp-project'); const listGcloudProjects = vi.fn(async () => [ { projectId: 'local-gcp-project', name: 'Local project' }, @@ -280,7 +299,6 @@ describe('setup Anthropic model step', () => { { prompts, env: {}, - runGcloudAuth, readGcloudProject, listGcloudProjects, healthCheck, @@ -288,7 +306,15 @@ describe('setup Anthropic model step', () => { ); expect(result.status).toBe('ready'); - expect(runGcloudAuth).toHaveBeenCalledWith(io.io); + expect(prompts.select).toHaveBeenCalledWith( + expect.objectContaining({ + message: expect.stringContaining('How should KTX authenticate with Google Vertex AI?'), + options: [ + { value: 'existing', label: 'Use existing gcloud/Application Default Credentials' }, + { value: 'back', label: 'Back' }, + ], + }), + ); expect(readGcloudProject).toHaveBeenCalled(); expect(listGcloudProjects).toHaveBeenCalled(); expect(prompts.text).not.toHaveBeenCalled(); @@ -303,6 +329,22 @@ describe('setup Anthropic model step', () => { ], }), ); + expect(prompts.select).toHaveBeenCalledWith( + expect.objectContaining({ + message: expect.stringContaining('Which Anthropic model should KTX use?'), + options: [ + { value: 'claude-opus-4-7', label: 'Claude Opus 4.7' }, + { value: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6' }, + { value: 'claude-opus-4-6', label: 'Claude Opus 4.6' }, + { value: 'claude-opus-4-5', label: 'Claude Opus 4.5' }, + { value: 'claude-haiku-4-5', label: 'Claude Haiku 4.5' }, + { value: 'claude-sonnet-4-5', label: 'Claude Sonnet 4.5' }, + { value: 'claude-opus-4-1', label: 'Claude Opus 4.1' }, + { value: 'manual', label: 'Enter a model ID manually' }, + { value: 'back', label: 'Back' }, + ], + }), + ); expect(healthCheck).toHaveBeenCalledWith({ backend: 'vertex', vertex: { project: 'local-gcp-project', location: 'us-east5' }, @@ -415,35 +457,6 @@ describe('setup Anthropic model step', () => { ); }); - it('runs only gcloud application-default login for Vertex AI auth', async () => { - const io = makeIo(); - const runGcloud = vi.fn(async () => ({ ok: true as const })); - - await expect(runKtxSetupGcloudApplicationDefaultAuth(io.io, runGcloud)).resolves.toEqual({ ok: true }); - - expect(runGcloud).toHaveBeenCalledTimes(1); - expect(runGcloud).toHaveBeenCalledWith(['auth', 'application-default', 'login'], expect.anything()); - expect(runGcloud).not.toHaveBeenCalledWith(['auth', 'login'], expect.anything()); - expect(io.stdout()).toContain('gcloud auth application-default login'); - expect(io.stdout()).not.toContain('gcloud auth login'); - }); - - it('indents gcloud auth output inside the setup gutter', async () => { - const io = makeIo(); - const runGcloud = vi.fn(async (_args: string[], commandIo: KtxCliIo) => { - commandIo.stdout.write('Your browser has been opened to visit:\n\n https://accounts.example/auth\n'); - commandIo.stderr.write('Credentials saved to file: [/tmp/application_default_credentials.json]\n'); - return { ok: true as const }; - }); - - await expect(runKtxSetupGcloudApplicationDefaultAuth(io.io, runGcloud)).resolves.toEqual({ ok: true }); - - expect(io.stdout()).toContain('│ Your browser has been opened to visit:'); - expect(io.stdout()).toContain('│ https://accounts.example/auth'); - expect(io.stderr()).toContain('│ Credentials saved to file: [/tmp/application_default_credentials.json]'); - expect(io.stdout()).not.toContain('\nYour browser has been opened'); - }); - it('explains common Vertex AI Forbidden health-check causes', async () => { const io = makeIo(); diff --git a/packages/cli/src/setup-models.ts b/packages/cli/src/setup-models.ts index bd05bd44..e4c7fcd2 100644 --- a/packages/cli/src/setup-models.ts +++ b/packages/cli/src/setup-models.ts @@ -1,4 +1,4 @@ -import { execFile, spawn } from 'node:child_process'; +import { execFile } from 'node:child_process'; import { writeFile } from 'node:fs/promises'; import { promisify } from 'node:util'; import { resolveLocalKtxLlmConfig } from '@ktx/context'; @@ -11,6 +11,7 @@ import { serializeKtxProjectConfig, } from '@ktx/context/project'; import { type KtxLlmConfig, type KtxLlmHealthCheckResult, runKtxLlmHealthCheck } from '@ktx/llm'; +import { createClackSpinner, type KtxCliSpinner } from './clack.js'; import type { KtxCliIo } from './cli-runtime.js'; import { withTextInputNavigation } from './prompt-navigation.js'; import { envCredentialReference, writeProjectLocalSecretReference } from './setup-secrets.js'; @@ -61,9 +62,9 @@ export interface KtxSetupModelDeps { prompts?: KtxSetupModelPromptAdapter; listModels?: (apiKey: string) => Promise; healthCheck?: (config: KtxLlmConfig) => Promise; - runGcloudAuth?: (io: KtxCliIo) => Promise; readGcloudProject?: () => Promise; listGcloudProjects?: () => Promise; + spinner?: () => KtxCliSpinner; } export const BUNDLED_ANTHROPIC_MODEL_REGISTRY_VERSION = '2026-05-07'; @@ -74,6 +75,16 @@ export const BUNDLED_ANTHROPIC_MODELS: AnthropicModelChoice[] = [ { id: 'claude-haiku-4-5', label: 'Claude Haiku 4.5', recommended: false }, ]; +const VERTEX_ANTHROPIC_MODELS: AnthropicModelChoice[] = [ + { id: 'claude-opus-4-7', label: 'Claude Opus 4.7', recommended: false }, + { id: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6', recommended: false }, + { id: 'claude-opus-4-6', label: 'Claude Opus 4.6', recommended: false }, + { id: 'claude-opus-4-5', label: 'Claude Opus 4.5', recommended: false }, + { id: 'claude-haiku-4-5', label: 'Claude Haiku 4.5', recommended: false }, + { id: 'claude-sonnet-4-5', label: 'Claude Sonnet 4.5', recommended: false }, + { id: 'claude-opus-4-1', label: 'Claude Opus 4.1', recommended: false }, +]; + const HIDDEN_ANTHROPIC_MODEL_PATTERNS = [ /^claude-sonnet-4$/i, /^claude-opus-4$/i, @@ -91,8 +102,8 @@ const ANTHROPIC_MODEL_PROMPT_CONTEXT = 'into semantic-layer sources and wiki context.'; const VERTEX_AUTH_PROMPT_CONTEXT = - 'KTX can use Google Cloud Application Default Credentials for local Vertex AI access. This opens the normal ' + - 'gcloud browser login flow and does not store Google credentials in ktx.yaml.'; + 'KTX uses Google Cloud Application Default Credentials for local Vertex AI access and does not store Google ' + + 'credentials in ktx.yaml. If needed, run gcloud auth application-default login before continuing.'; const VERTEX_PROJECT_PROMPT_CONTEXT = 'KTX stores the selected Google Cloud project ID in ktx.yaml and uses Application Default Credentials for ' + 'access. Project visibility depends on the signed-in Google account and organization permissions.'; @@ -137,94 +148,17 @@ type VertexConfigChoice = } | { status: 'back' | 'missing-input' }; -type VertexAuthChoice = { status: 'ready' } | { status: 'back' | 'missing-input' }; +type VertexAuthChoice = { status: 'ready' } | { status: 'back' }; -export type GcloudAuthResult = { ok: true } | { ok: false; message: string }; interface GcloudProjectChoice { projectId: string; name?: string; } -type GcloudCommandRunner = (args: string[], io: KtxCliIo) => Promise; function createPromptAdapter(): KtxSetupModelPromptAdapter { return createKtxSetupPromptAdapter({ selectCancelValue: 'back' }); } -function createIndentedCommandIo(io: KtxCliIo): KtxCliIo { - const indentedWriter = (write: (chunk: string) => void) => { - let atLineStart = true; - return (chunk: string) => { - for (const char of chunk) { - if (atLineStart) { - write('│ '); - atLineStart = false; - } - write(char); - if (char === '\n') { - atLineStart = true; - } - } - }; - }; - - return { - stdout: { - isTTY: io.stdout.isTTY, - columns: io.stdout.columns, - write: indentedWriter((chunk) => io.stdout.write(chunk)), - }, - stderr: { - write: indentedWriter((chunk) => io.stderr.write(chunk)), - }, - }; -} - -function runInteractiveGcloud(args: string[], io: KtxCliIo): Promise { - return new Promise((resolve) => { - let settled = false; - const child = spawn('gcloud', args, { stdio: ['inherit', 'pipe', 'pipe'] }); - child.stdout?.on('data', (chunk: Buffer) => { - io.stdout.write(chunk.toString('utf8')); - }); - child.stderr?.on('data', (chunk: Buffer) => { - io.stderr.write(chunk.toString('utf8')); - }); - child.on('error', (error: NodeJS.ErrnoException) => { - if (settled) { - return; - } - settled = true; - if (error.code === 'ENOENT') { - resolve({ ok: false, message: 'gcloud CLI was not found on PATH.' }); - return; - } - resolve({ ok: false, message: error.message }); - }); - child.on('close', (code, signal) => { - if (settled) { - return; - } - settled = true; - if (code === 0) { - resolve({ ok: true }); - return; - } - resolve({ - ok: false, - message: signal ? `gcloud exited after signal ${signal}.` : `gcloud exited with code ${code ?? 'unknown'}.`, - }); - }); - }); -} - -export async function runKtxSetupGcloudApplicationDefaultAuth( - io: KtxCliIo, - runGcloud: GcloudCommandRunner = runInteractiveGcloud, -): Promise { - io.stdout.write('│ Running gcloud auth application-default login...\n'); - return await runGcloud(['auth', 'application-default', 'login'], createIndentedCommandIo(io)); -} - async function defaultReadGcloudProject(): Promise { try { const { stdout } = await execFileAsync('gcloud', ['config', 'get-value', 'project'], { encoding: 'utf8' }); @@ -374,6 +308,53 @@ function buildVertexHealthConfig(vertex: { project?: string; location: string }, }; } +type LlmHealthProvider = 'Anthropic API' | 'Vertex AI'; + +function llmHealthCheckStartText(provider: LlmHealthProvider, model: string): string { + return `Checking ${provider} LLM (${model}).`; +} + +function startLlmHealthCheckProgress( + spinner: KtxCliSpinner, + message: string, +): { succeed(msg: string): void; fail(msg: string): void } { + spinner.start(message); + return { + succeed(msg: string) { + spinner.stop(msg); + }, + fail(msg: string) { + spinner.error(msg); + }, + }; +} + +async function runLlmHealthCheckWithProgress( + config: KtxLlmConfig, + provider: LlmHealthProvider, + model: string, + healthCheck: (config: KtxLlmConfig) => Promise, + deps: KtxSetupModelDeps, +): Promise { + const progress = startLlmHealthCheckProgress( + (deps.spinner ?? createClackSpinner)(), + llmHealthCheckStartText(provider, model), + ); + let health: KtxLlmHealthCheckResult; + try { + health = await healthCheck(config); + } catch (error) { + progress.fail('LLM test failed'); + throw error; + } + if (health.ok) { + progress.succeed(`LLM test passed (${provider}, ${model})`); + } else { + progress.fail('LLM test failed'); + } + return health; +} + function formatVertexHealthFailure(message: string, vertex: { project?: string; location: string }): string { const trimmed = message.trim() || 'unknown error'; if (!/(forbidden|permission|permission_denied|403)/i.test(trimmed)) { @@ -516,7 +497,6 @@ async function chooseBackend( async function chooseVertexAuth( args: KtxSetupModelArgs, - io: KtxCliIo, deps: KtxSetupModelDeps, ): Promise { if (args.inputMode === 'disabled' || args.vertexProject || args.vertexLocation) { @@ -527,7 +507,6 @@ async function chooseVertexAuth( const choice = await prompts.select({ message: `How should KTX authenticate with Google Vertex AI?\n\n${VERTEX_AUTH_PROMPT_CONTEXT}`, options: [ - { value: 'gcloud', label: 'Run gcloud Application Default Credentials login' }, { value: 'existing', label: 'Use existing gcloud/Application Default Credentials' }, { value: 'back', label: 'Back' }, ], @@ -535,15 +514,6 @@ async function chooseVertexAuth( if (choice === 'back') { return { status: 'back' }; } - if (choice !== 'gcloud') { - return { status: 'ready' }; - } - - const result = await (deps.runGcloudAuth ?? runKtxSetupGcloudApplicationDefaultAuth)(io); - if (!result.ok) { - io.stderr.write(`gcloud authentication failed: ${result.message}\n`); - return { status: 'missing-input' }; - } return { status: 'ready' }; } @@ -799,7 +769,7 @@ async function chooseVertexModel(args: KtxSetupModelArgs, io: KtxCliIo, deps: Kt return { status: 'missing-input' }; } - const selectableModels = BUNDLED_ANTHROPIC_MODELS.filter(isSelectableAnthropicModel); + const selectableModels = VERTEX_ANTHROPIC_MODELS.filter(isSelectableAnthropicModel); const prompts = deps.prompts ?? createPromptAdapter(); const choice = await prompts.select({ message: `Which Anthropic model should KTX use?\n\n${ANTHROPIC_MODEL_PROMPT_CONTEXT}`, @@ -901,7 +871,7 @@ export async function runKtxSetupAnthropicModelStep( : attemptArgs; if (backendChoice.backend === 'vertex') { - const auth = await chooseVertexAuth(backendArgs, io, deps); + const auth = await chooseVertexAuth(backendArgs, deps); if (auth.status === 'back' && backendChoice.prompted) { attemptArgs = buildInteractiveRetryArgs(args); continue; @@ -931,7 +901,13 @@ export async function runKtxSetupAnthropicModelStep( return { status: model.status, projectDir: args.projectDir }; } - const health = await healthCheck(buildVertexHealthConfig(vertex.values, model.model)); + const health = await runLlmHealthCheckWithProgress( + buildVertexHealthConfig(vertex.values, model.model), + 'Vertex AI', + model.model, + healthCheck, + deps, + ); if (health.ok) { await persistLlmConfig(args.projectDir, { backend: 'vertex', vertex: vertex.refs }, model.model); io.stdout.write(`│ LLM ready: yes (${model.model})\n`); @@ -973,7 +949,13 @@ export async function runKtxSetupAnthropicModelStep( return { status: model.status, projectDir: args.projectDir }; } - const health = await healthCheck(buildAnthropicHealthConfig(credential.value, model.model)); + const health = await runLlmHealthCheckWithProgress( + buildAnthropicHealthConfig(credential.value, model.model), + 'Anthropic API', + model.model, + healthCheck, + deps, + ); if (health.ok) { await persistLlmConfig(args.projectDir, { backend: 'anthropic', credentialRef: credential.ref }, model.model); io.stdout.write(`│ LLM ready: yes (${model.model})\n`); From 9ecb8cb119411ab6808cd9c3f1806fc817fa17b3 Mon Sep 17 00:00:00 2001 From: Luca Martial <48870843+luca-martial@users.noreply.github.com> Date: Wed, 13 May 2026 17:22:59 -0400 Subject: [PATCH 3/7] feat(cli): add edit flow for setup connections (#77) * feat(cli): add edit flow for primary database connections in setup Allow users to edit existing primary database connections during setup instead of only adding new ones. Preselects existing values (URL, schemas, tables) so users can adjust without re-entering everything. Co-Authored-By: Claude Opus 4.6 (1M context) * feat(cli): add edit flow for context source connections in setup Allow users to edit existing context source connections during setup. Preselects existing values (URLs, credentials, repo details) and offers a "Keep existing credential" option for sensitive fields. Co-Authored-By: Claude Opus 4.6 (1M context) * fix(cli): rename "Add more" to "Add additional" in primary sources menu Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Claude Opus 4.6 (1M context) --- packages/cli/src/setup-databases.test.ts | 404 ++++++++++++++++++- packages/cli/src/setup-databases.ts | 490 +++++++++++++++++++---- packages/cli/src/setup-sources.test.ts | 313 +++++++++++++++ packages/cli/src/setup-sources.ts | 488 +++++++++++++++++++--- 4 files changed, 1553 insertions(+), 142 deletions(-) diff --git a/packages/cli/src/setup-databases.test.ts b/packages/cli/src/setup-databases.test.ts index d010a908..fa4ca3f2 100644 --- a/packages/cli/src/setup-databases.test.ts +++ b/packages/cli/src/setup-databases.test.ts @@ -240,8 +240,9 @@ describe('setup databases step', () => { expect(prompts.select).toHaveBeenCalledWith({ message: 'Configure PostgreSQL', options: [ - { value: 'existing:warehouse', label: 'Use existing PostgreSQL connection: warehouse' }, - { value: 'new', label: 'Add new PostgreSQL connection' }, + { value: 'existing:warehouse', label: 'Keep existing PostgreSQL connection: warehouse' }, + { value: 'edit:warehouse', label: 'Edit PostgreSQL connection: warehouse' }, + { value: 'new', label: 'Add another PostgreSQL connection' }, { value: 'back', label: 'Back' }, ], }); @@ -564,7 +565,8 @@ describe('setup databases step', () => { message: 'Primary sources already configured: warehouse\nWhat would you like to do?', options: [ { value: 'continue', label: 'Continue to knowledge sources' }, - { value: 'add', label: 'Add another primary source' }, + { value: 'edit', label: 'Edit an existing primary source' }, + { value: 'add', label: 'Add additional primary sources' }, ], }); expect(testConnection).not.toHaveBeenCalled(); @@ -608,11 +610,16 @@ describe('setup databases step', () => { connectionIds: ['warehouse', 'mysql-warehouse'], }); expect(prompts.multiselect).toHaveBeenCalledTimes(1); + expect(prompts.multiselect).toHaveBeenCalledWith(expect.objectContaining({ + initialValues: ['postgres'], + required: true, + })); expect(prompts.select).toHaveBeenCalledWith({ message: 'Primary sources already configured: warehouse\nWhat would you like to do?', options: [ { value: 'continue', label: 'Continue to knowledge sources' }, - { value: 'add', label: 'Add another primary source' }, + { value: 'edit', label: 'Edit an existing primary source' }, + { value: 'add', label: 'Add additional primary sources' }, ], }); expect(testConnection).toHaveBeenCalledTimes(1); @@ -642,11 +649,16 @@ describe('setup databases step', () => { connectionIds: ['postgres-warehouse', 'mysql-warehouse'], }); expect(prompts.multiselect).toHaveBeenCalledTimes(2); + expect(prompts.multiselect).toHaveBeenNthCalledWith(2, expect.objectContaining({ + initialValues: ['postgres'], + required: true, + })); expect(prompts.select).toHaveBeenCalledWith({ message: 'Primary sources already configured: postgres-warehouse\nWhat would you like to do?', options: [ { value: 'continue', label: 'Continue to knowledge sources' }, - { value: 'add', label: 'Add another primary source' }, + { value: 'edit', label: 'Edit an existing primary source' }, + { value: 'add', label: 'Add additional primary sources' }, ], }); const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); @@ -675,12 +687,17 @@ describe('setup databases step', () => { connectionIds: ['postgres-warehouse'], }); expect(prompts.multiselect).toHaveBeenCalledTimes(2); + expect(prompts.multiselect).toHaveBeenNthCalledWith(2, expect.objectContaining({ + initialValues: ['postgres'], + required: true, + })); expect(io.stdout()).not.toContain('KTX cannot work without at least one primary source'); expect(prompts.select).toHaveBeenNthCalledWith(2, { message: 'Primary sources already configured: postgres-warehouse\nWhat would you like to do?', options: [ { value: 'continue', label: 'Continue to knowledge sources' }, - { value: 'add', label: 'Add another primary source' }, + { value: 'edit', label: 'Edit an existing primary source' }, + { value: 'add', label: 'Add additional primary sources' }, ], }); }); @@ -715,16 +732,389 @@ describe('setup databases step', () => { ); expect(result).toEqual({ status: 'ready', projectDir: tempDir, connectionIds: ['warehouse'] }); + expect(prompts.multiselect).toHaveBeenCalledWith(expect.objectContaining({ + initialValues: ['postgres'], + required: true, + })); expect(io.stdout()).not.toContain('KTX cannot work without at least one primary source'); expect(prompts.select).toHaveBeenNthCalledWith(2, { message: 'Primary sources already configured: warehouse\nWhat would you like to do?', options: [ { value: 'continue', label: 'Continue to knowledge sources' }, - { value: 'add', label: 'Add another primary source' }, + { value: 'edit', label: 'Edit an existing primary source' }, + { value: 'add', label: 'Add additional primary sources' }, ], }); }); + it('returns from primary source edit selection back to the configured source menu', async () => { + await writeFile( + join(tempDir, 'ktx.yaml'), + [ + 'project: warehouse', + 'connections:', + ' warehouse:', + ' driver: postgres', + ' url: env:DATABASE_URL', + 'setup:', + ' database_connection_ids:', + ' - warehouse', + '', + ].join('\n'), + 'utf-8', + ); + await writeKtxSetupState(tempDir, { completed_steps: ['databases'] }); + const prompts = makePromptAdapter({ + selectValues: ['edit', 'back', 'continue'], + }); + const testConnection = vi.fn(async () => 0); + const scanConnection = vi.fn(async () => 0); + + const result = await runKtxSetupDatabasesStep( + { projectDir: tempDir, inputMode: 'auto', skipDatabases: false, databaseSchemas: [] }, + makeIo().io, + { prompts, testConnection, scanConnection }, + ); + + expect(result).toEqual({ status: 'ready', projectDir: tempDir, connectionIds: ['warehouse'] }); + expect(prompts.select).toHaveBeenNthCalledWith(2, { + message: 'Primary source to edit', + options: [ + { value: 'warehouse', label: 'warehouse (PostgreSQL)' }, + { value: 'back', label: 'Back' }, + ], + }); + expect(prompts.select).toHaveBeenNthCalledWith(3, { + message: 'Primary sources already configured: warehouse\nWhat would you like to do?', + options: [ + { value: 'continue', label: 'Continue to knowledge sources' }, + { value: 'edit', label: 'Edit an existing primary source' }, + { value: 'add', label: 'Add additional primary sources' }, + ], + }); + expect(testConnection).not.toHaveBeenCalled(); + expect(scanConnection).not.toHaveBeenCalled(); + }); + + it('reruns table selection after editing schema scope so stale enabled tables are removed', async () => { + await writeFile( + join(tempDir, 'ktx.yaml'), + [ + 'project: warehouse', + 'connections:', + ' warehouse:', + ' driver: postgres', + ' url: env:DATABASE_URL', + ' schemas:', + ' - public', + ' enabled_tables:', + ' - public.orders', + 'setup:', + ' database_connection_ids:', + ' - warehouse', + '', + ].join('\n'), + 'utf-8', + ); + await writeKtxSetupState(tempDir, { completed_steps: ['databases'] }); + const prompts = makePromptAdapter({ + textValues: ['env:DATABASE_URL'], + multiselectValues: [['analytics']], + }); + let primaryMenuCount = 0; + vi.mocked(prompts.select).mockImplementation(async (options) => { + if (options.message === 'Primary sources already configured: warehouse\nWhat would you like to do?') { + primaryMenuCount += 1; + return primaryMenuCount === 1 ? 'edit' : 'continue'; + } + if (options.message === 'Primary source to edit') return 'warehouse'; + if (options.message === 'How do you want to connect to PostgreSQL?') return 'url'; + return 'back'; + }); + const testConnection = vi.fn(async () => 0); + const scanConnection = vi.fn(async () => 0); + const listSchemas = vi.fn(async () => ['analytics', 'public']); + const listTables = vi.fn(async () => [{ schema: 'analytics', name: 'customers', kind: 'table' as const }]); + + const result = await runKtxSetupDatabasesStep( + { projectDir: tempDir, inputMode: 'auto', skipDatabases: false, databaseSchemas: [] }, + makeIo().io, + { prompts, testConnection, scanConnection, listSchemas, listTables }, + ); + + expect(result).toEqual({ status: 'ready', projectDir: tempDir, connectionIds: ['warehouse'] }); + expect(prompts.text).toHaveBeenCalledWith({ + message: textInputPrompt('PostgreSQL connection URL'), + placeholder: 'env:DATABASE_URL', + initialValue: 'env:DATABASE_URL', + }); + expect(listTables).toHaveBeenCalledWith(tempDir, 'warehouse'); + expect(testConnection).toHaveBeenCalledWith(tempDir, 'warehouse', expect.anything()); + expect(scanConnection).toHaveBeenCalledWith(tempDir, 'warehouse', expect.anything()); + const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + expect(config.connections.warehouse).toMatchObject({ + schemas: ['analytics'], + enabled_tables: ['analytics.customers'], + }); + }); + + it('preselects existing schema and table choices when editing a primary source', async () => { + await writeFile( + join(tempDir, 'ktx.yaml'), + [ + 'project: warehouse', + 'connections:', + ' warehouse:', + ' driver: postgres', + ' url: env:DATABASE_URL', + ' schemas:', + ' - public', + ' enabled_tables:', + ' - public.customers', + ' - public.orders', + 'setup:', + ' database_connection_ids:', + ' - warehouse', + '', + ].join('\n'), + 'utf-8', + ); + await writeKtxSetupState(tempDir, { completed_steps: ['databases'] }); + const prompts = makePromptAdapter({ + textValues: ['env:DATABASE_URL'], + multiselectValues: [['public'], ['public.customers', 'public.orders']], + }); + let primaryMenuCount = 0; + vi.mocked(prompts.select).mockImplementation(async (options) => { + if (options.message === 'Primary sources already configured: warehouse\nWhat would you like to do?') { + primaryMenuCount += 1; + return primaryMenuCount === 1 ? 'edit' : 'continue'; + } + if (options.message === 'Primary source to edit') return 'warehouse'; + if (options.message === 'How do you want to connect to PostgreSQL?') return 'url'; + if (options.message.startsWith('Tables found in selected schemas')) return 'customize'; + return 'back'; + }); + const listSchemas = vi.fn(async () => ['orbit_analytics', 'orbit_raw', 'public']); + const listTables = vi.fn(async () => [ + { schema: 'public', name: 'customers', kind: 'table' as const }, + { schema: 'public', name: 'orders', kind: 'table' as const }, + { schema: 'public', name: 'products', kind: 'table' as const }, + ]); + + const result = await runKtxSetupDatabasesStep( + { projectDir: tempDir, inputMode: 'auto', skipDatabases: false, databaseSchemas: [] }, + makeIo().io, + { + prompts, + testConnection: vi.fn(async () => 0), + scanConnection: vi.fn(async () => 0), + listSchemas, + listTables, + }, + ); + + expect(result).toEqual({ status: 'ready', projectDir: tempDir, connectionIds: ['warehouse'] }); + expect(prompts.multiselect).toHaveBeenNthCalledWith(1, { + message: expect.stringContaining('PostgreSQL schemas to scan'), + options: [ + { value: 'orbit_analytics', label: 'orbit_analytics' }, + { value: 'orbit_raw', label: 'orbit_raw' }, + { value: 'public', label: 'public' }, + ], + initialValues: ['public'], + required: true, + }); + expect(prompts.multiselect).toHaveBeenNthCalledWith(2, { + message: expect.stringContaining('Tables to enable for warehouse'), + options: [ + { value: 'public.customers', label: 'public.customers' }, + { value: 'public.orders', label: 'public.orders' }, + { value: 'public.products', label: 'public.products' }, + ], + initialValues: ['public.customers', 'public.orders'], + required: true, + }); + const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + expect(config.connections.warehouse).toMatchObject({ + schemas: ['public'], + enabled_tables: ['public.customers', 'public.orders'], + }); + }); + + it('returns to the configured primary menu when backing out of schema review during edit', async () => { + await writeFile( + join(tempDir, 'ktx.yaml'), + [ + 'project: warehouse', + 'connections:', + ' warehouse:', + ' driver: postgres', + ' url: env:DATABASE_URL', + ' schemas:', + ' - public', + ' enabled_tables:', + ' - public.orders', + 'setup:', + ' database_connection_ids:', + ' - warehouse', + '', + ].join('\n'), + 'utf-8', + ); + await writeKtxSetupState(tempDir, { completed_steps: ['databases'] }); + const prompts = makePromptAdapter({ + textValues: ['env:DATABASE_URL'], + multiselectValues: [['back']], + }); + let primaryMenuCount = 0; + vi.mocked(prompts.select).mockImplementation(async (options) => { + if (options.message === 'Primary sources already configured: warehouse\nWhat would you like to do?') { + primaryMenuCount += 1; + return primaryMenuCount === 1 ? 'edit' : 'continue'; + } + if (options.message === 'Primary source to edit') return 'warehouse'; + if (options.message === 'How do you want to connect to PostgreSQL?') return 'url'; + return 'back'; + }); + const testConnection = vi.fn(async () => 0); + const scanConnection = vi.fn(async () => 0); + const listSchemas = vi.fn(async () => ['analytics', 'public']); + const listTables = vi.fn(async () => [{ schema: 'analytics', name: 'customers', kind: 'table' as const }]); + + const result = await runKtxSetupDatabasesStep( + { projectDir: tempDir, inputMode: 'auto', skipDatabases: false, databaseSchemas: [] }, + makeIo().io, + { prompts, testConnection, scanConnection, listSchemas, listTables }, + ); + + expect(result).toEqual({ status: 'ready', projectDir: tempDir, connectionIds: ['warehouse'] }); + expect(primaryMenuCount).toBe(2); + expect(testConnection).toHaveBeenCalledWith(tempDir, 'warehouse', expect.anything()); + expect(scanConnection).not.toHaveBeenCalled(); + expect(listTables).not.toHaveBeenCalled(); + const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + expect(config.connections.warehouse).toMatchObject({ + url: 'env:DATABASE_URL', + schemas: ['public'], + enabled_tables: ['public.orders'], + }); + }); + + it('returns to the configured primary menu when backing out of table review during edit', async () => { + await writeFile( + join(tempDir, 'ktx.yaml'), + [ + 'project: warehouse', + 'connections:', + ' warehouse:', + ' driver: postgres', + ' url: env:DATABASE_URL', + ' schemas:', + ' - public', + ' enabled_tables:', + ' - public.orders', + 'setup:', + ' database_connection_ids:', + ' - warehouse', + '', + ].join('\n'), + 'utf-8', + ); + await writeKtxSetupState(tempDir, { completed_steps: ['databases'] }); + const prompts = makePromptAdapter({ textValues: ['env:DATABASE_URL'] }); + let primaryMenuCount = 0; + vi.mocked(prompts.select).mockImplementation(async (options) => { + if (options.message === 'Primary sources already configured: warehouse\nWhat would you like to do?') { + primaryMenuCount += 1; + return primaryMenuCount === 1 ? 'edit' : 'continue'; + } + if (options.message === 'Primary source to edit') return 'warehouse'; + if (options.message === 'How do you want to connect to PostgreSQL?') return 'url'; + if (options.message.startsWith('Tables found in selected schemas')) return 'back'; + return 'back'; + }); + const testConnection = vi.fn(async () => 0); + const scanConnection = vi.fn(async () => 0); + const listSchemas = vi.fn(async () => ['public']); + const listTables = vi.fn(async () => [ + { schema: 'public', name: 'customers', kind: 'table' as const }, + { schema: 'public', name: 'orders', kind: 'table' as const }, + ]); + + const result = await runKtxSetupDatabasesStep( + { projectDir: tempDir, inputMode: 'auto', skipDatabases: false, databaseSchemas: [] }, + makeIo().io, + { prompts, testConnection, scanConnection, listSchemas, listTables }, + ); + + expect(result).toEqual({ status: 'ready', projectDir: tempDir, connectionIds: ['warehouse'] }); + expect(primaryMenuCount).toBe(2); + expect(listTables).toHaveBeenCalledWith(tempDir, 'warehouse'); + expect(scanConnection).not.toHaveBeenCalled(); + const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + expect(config.connections.warehouse).toMatchObject({ + url: 'env:DATABASE_URL', + schemas: ['public'], + enabled_tables: ['public.orders'], + }); + }); + + it('restores an existing primary source edit when the follow-up scan fails', async () => { + await writeFile( + join(tempDir, 'ktx.yaml'), + [ + 'project: warehouse', + 'connections:', + ' warehouse:', + ' driver: postgres', + ' url: env:DATABASE_URL', + ' schemas:', + ' - public', + ' enabled_tables:', + ' - public.orders', + 'setup:', + ' database_connection_ids:', + ' - warehouse', + '', + ].join('\n'), + 'utf-8', + ); + await writeKtxSetupState(tempDir, { completed_steps: ['databases'] }); + const prompts = makePromptAdapter({ + textValues: ['env:DATABASE_URL'], + multiselectValues: [['public']], + }); + vi.mocked(prompts.select).mockImplementation(async (options) => { + if (options.message === 'Primary sources already configured: warehouse\nWhat would you like to do?') return 'edit'; + if (options.message === 'Primary source to edit') return 'warehouse'; + if (options.message === 'How do you want to connect to PostgreSQL?') return 'url'; + if (options.message.startsWith('Tables found in selected schemas')) return 'all'; + return 'back'; + }); + const listTables = vi.fn(async () => [ + { schema: 'public', name: 'customers', kind: 'table' as const }, + { schema: 'public', name: 'orders', kind: 'table' as const }, + ]); + + const result = await runKtxSetupDatabasesStep( + { projectDir: tempDir, inputMode: 'auto', skipDatabases: false, databaseSchemas: [] }, + makeIo().io, + { + prompts, + testConnection: vi.fn(async () => 0), + scanConnection: vi.fn(async () => 1), + listTables, + }, + ); + + expect(result).toEqual({ status: 'failed', projectDir: tempDir }); + const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + expect(config.connections.warehouse).toMatchObject({ + enabled_tables: ['public.orders'], + }); + }); + it('lets Escape from connection fields return to connection method selection', async () => { const prompts = makePromptAdapter({ selectValues: ['fields', 'url'], diff --git a/packages/cli/src/setup-databases.ts b/packages/cli/src/setup-databases.ts index 5b5b5f8a..9db80689 100644 --- a/packages/cli/src/setup-databases.ts +++ b/packages/cli/src/setup-databases.ts @@ -176,6 +176,7 @@ const SCOPE_DISCOVERY_SPECS: Partial; +type ConnectionSetupStatus = 'ready' | 'back' | 'failed'; const DRIVER_CONNECTION_DEFAULTS: Record = { postgres: { port: '5432' }, @@ -227,6 +228,16 @@ function unique(values: string[]): string[] { return [...new Set(values.filter((value) => value.trim().length > 0))]; } +function stringConfigField(connection: KtxProjectConnectionConfig | undefined, field: string): string | undefined { + const value = connection?.[field]; + return typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined; +} + +function numberConfigField(connection: KtxProjectConnectionConfig | undefined, field: string): number | undefined { + const value = connection?.[field]; + return typeof value === 'number' && Number.isFinite(value) ? value : undefined; +} + function historicSqlConfigRecord(connection: KtxProjectConnectionConfig | undefined): Record | null { const historicSql = connection?.historicSql; return historicSql && typeof historicSql === 'object' && !Array.isArray(historicSql) @@ -454,6 +465,18 @@ function configuredPrimaryConnectionIds( .sort((left, right) => left.localeCompare(right)); } +function configuredPrimaryDrivers( + connections: Record, + connectionIds: string[], +): KtxSetupDatabaseDriver[] { + const configured = new Set( + connectionIds + .map((connectionId) => normalizeDriver(connections[connectionId]?.driver)) + .filter((driver): driver is KtxSetupDatabaseDriver => driver !== null), + ); + return DRIVER_OPTIONS.map((option) => option.value).filter((driver) => configured.has(driver)); +} + function configuredPrimarySourcesPrompt(connectionIds: string[]): { message: string; options: Array<{ value: string; label: string }>; @@ -462,7 +485,8 @@ function configuredPrimarySourcesPrompt(connectionIds: string[]): { message: `Primary sources already configured: ${connectionIds.join(', ')}\nWhat would you like to do?`, options: [ { value: 'continue', label: 'Continue to knowledge sources' }, - { value: 'add', label: 'Add another primary source' }, + { value: 'edit', label: 'Edit an existing primary source' }, + { value: 'add', label: 'Add additional primary sources' }, ], }; } @@ -552,23 +576,40 @@ async function buildFieldsConnectionConfig(input: { connectionId: string; args: KtxSetupDatabasesArgs; prompts: KtxSetupDatabasesPromptAdapter; + existingConnection?: KtxProjectConnectionConfig; }): Promise { const label = driverLabel(input.driver); const defaults = DRIVER_CONNECTION_DEFAULTS[input.driver]; - const host = await promptText(input.prompts, `${label} host`, 'localhost'); + const host = await promptText( + input.prompts, + `${label} host`, + stringConfigField(input.existingConnection, 'host') ?? 'localhost', + ); if (host === undefined) return 'back'; if (!host) return null; - const portStr = await promptText(input.prompts, `${label} port`, defaults.port); + const portStr = await promptText( + input.prompts, + `${label} port`, + String(numberConfigField(input.existingConnection, 'port') ?? defaults.port), + ); if (portStr === undefined) return 'back'; const port = Number(portStr || defaults.port); - const database = await promptText(input.prompts, `${label} database name`); + const database = await promptText( + input.prompts, + `${label} database name`, + stringConfigField(input.existingConnection, 'database'), + ); if (database === undefined) return 'back'; if (!database) return null; - const username = await promptText(input.prompts, `${label} username`); + const username = await promptText( + input.prompts, + `${label} username`, + stringConfigField(input.existingConnection, 'username'), + ); if (username === undefined) return 'back'; if (!username) return null; @@ -583,6 +624,7 @@ async function buildFieldsConnectionConfig(input: { }); if (credentialResult === 'back') return 'back'; if (credentialResult) passwordRef = credentialResult; + if (!credentialResult) passwordRef = stringConfigField(input.existingConnection, 'password'); } return { @@ -601,9 +643,14 @@ async function buildPastedUrlConnectionConfig(input: { connectionId: string; args: KtxSetupDatabasesArgs; prompts: KtxSetupDatabasesPromptAdapter; + existingConnection?: KtxProjectConnectionConfig; }): Promise { const label = driverLabel(input.driver); - const rawUrl = await promptText(input.prompts, `${label} connection URL`); + const rawUrl = await promptText( + input.prompts, + `${label} connection URL`, + stringConfigField(input.existingConnection, 'url'), + ); if (rawUrl === undefined) return 'back'; if (!rawUrl) return null; @@ -642,6 +689,7 @@ async function buildUrlConnectionConfig(input: { connectionId: string; args: KtxSetupDatabasesArgs; prompts: KtxSetupDatabasesPromptAdapter; + existingConnection?: KtxProjectConnectionConfig; }): Promise { if (input.args.inputMode === 'disabled' && !input.args.databaseUrl) return null; @@ -689,6 +737,7 @@ async function buildConnectionConfig(input: { connectionId: string; args: KtxSetupDatabasesArgs; prompts: KtxSetupDatabasesPromptAdapter; + existingConnection?: KtxProjectConnectionConfig; }): Promise { const { driver, args, prompts } = input; if (driver === 'sqlite') { @@ -698,22 +747,37 @@ async function buildConnectionConfig(input: { (await promptText( prompts, 'SQLite database file\nEnter a relative or absolute path, for example ./warehouse.sqlite.', + stringConfigField(input.existingConnection, 'path'), )); if (path === undefined) return 'back'; return path ? { driver: 'sqlite', path } : null; } if (driver === 'postgres' || driver === 'mysql' || driver === 'clickhouse' || driver === 'sqlserver') { - return await buildUrlConnectionConfig({ driver, connectionId: input.connectionId, args, prompts }); + return await buildUrlConnectionConfig({ + driver, + connectionId: input.connectionId, + args, + prompts, + existingConnection: input.existingConnection, + }); } if (driver === 'bigquery') { - const datasetId = await promptText(prompts, 'BigQuery dataset\nFor example analytics.'); + const datasetId = await promptText( + prompts, + 'BigQuery dataset\nFor example analytics.', + stringConfigField(input.existingConnection, 'dataset_id'), + ); if (datasetId === undefined) return 'back'; - const credentialsPath = await promptText(prompts, 'Path to service account JSON file'); + const credentialsPath = await promptText( + prompts, + 'Path to service account JSON file', + stringConfigField(input.existingConnection, 'credentials_json'), + ); if (credentialsPath === undefined) return 'back'; const location = await promptText( prompts, 'BigQuery location\nPress Enter for US, or enter a location like EU.', - 'US', + stringConfigField(input.existingConnection, 'location') ?? 'US', ); if (location === undefined) return 'back'; if (!datasetId || !credentialsPath) return null; @@ -725,19 +789,35 @@ async function buildConnectionConfig(input: { }; } if (driver === 'snowflake') { - const account = await promptText(prompts, 'Snowflake account identifier'); + const account = await promptText( + prompts, + 'Snowflake account identifier', + stringConfigField(input.existingConnection, 'account'), + ); if (account === undefined) return 'back'; - const warehouse = await promptText(prompts, 'Snowflake warehouse\nFor example ANALYTICS_WH.'); + const warehouse = await promptText( + prompts, + 'Snowflake warehouse\nFor example ANALYTICS_WH.', + stringConfigField(input.existingConnection, 'warehouse'), + ); if (warehouse === undefined) return 'back'; - const database = await promptText(prompts, 'Snowflake database name'); + const database = await promptText( + prompts, + 'Snowflake database name', + stringConfigField(input.existingConnection, 'database'), + ); if (database === undefined) return 'back'; const schemaName = await promptText( prompts, 'Snowflake schema\nPress Enter for PUBLIC, or enter a schema name.', - 'PUBLIC', + stringConfigField(input.existingConnection, 'schema_name') ?? 'PUBLIC', ); if (schemaName === undefined) return 'back'; - const username = await promptText(prompts, 'Snowflake username'); + const username = await promptText( + prompts, + 'Snowflake username', + stringConfigField(input.existingConnection, 'username'), + ); if (username === undefined) return 'back'; const passwordRef = await promptCredential({ prompts, @@ -747,9 +827,14 @@ async function buildConnectionConfig(input: { secretName: 'password', // pragma: allowlist secret }); if (passwordRef === 'back') return 'back'; // pragma: allowlist secret - const role = await promptText(prompts, 'Snowflake role (optional)\nPress Enter to skip.'); + const role = await promptText( + prompts, + 'Snowflake role (optional)\nPress Enter to skip.', + stringConfigField(input.existingConnection, 'role'), + ); if (role === undefined) return 'back'; - if (!account || !warehouse || !database || !schemaName || !username || !passwordRef) return null; + const resolvedPasswordRef = passwordRef ?? stringConfigField(input.existingConnection, 'password'); + if (!account || !warehouse || !database || !schemaName || !username || !resolvedPasswordRef) return null; return { driver: 'snowflake', authMethod: 'password', @@ -758,7 +843,7 @@ async function buildConnectionConfig(input: { database, schema_name: schemaName, username, - password: passwordRef, + password: resolvedPasswordRef, ...(role ? { role } : {}), }; } @@ -1096,6 +1181,59 @@ async function writeConnectionConfig(input: { } } +async function createConnectionConfigRollback(projectDir: string, connectionId: string): Promise<() => Promise> { + const project = await loadKtxProject({ projectDir }); + const previousConnection = project.config.connections[connectionId]; + const hadPreviousConnection = previousConnection !== undefined; + return async () => { + const latest = await loadKtxProject({ projectDir }); + const connections = { ...latest.config.connections }; + if (hadPreviousConnection) { + connections[connectionId] = previousConnection; + } else { + delete connections[connectionId]; + } + await writeFile( + latest.configPath, + serializeKtxProjectConfig({ + ...latest.config, + connections, + }), + 'utf-8', + ); + }; +} + +function withExistingPrimaryEditPromptDefaults(input: { + previous: KtxProjectConnectionConfig; + next: KtxProjectConnectionConfig; + driver: KtxSetupDatabaseDriver; +}): KtxProjectConnectionConfig { + const merged: KtxProjectConnectionConfig = { ...input.next }; + const spec = SCOPE_DISCOVERY_SPECS[input.driver]; + if (spec) { + const nextArray = input.next[spec.configArrayField]; + const previousArray = input.previous[spec.configArrayField]; + if ( + !(Array.isArray(nextArray) && nextArray.length > 0) && + Array.isArray(previousArray) && + previousArray.length > 0 + ) { + delete merged[spec.configSingleField]; + merged[spec.configArrayField] = previousArray; + } else if (!Object.hasOwn(input.next, spec.configArrayField) && !Object.hasOwn(input.next, spec.configSingleField)) { + const previousSingle = input.previous[spec.configSingleField]; + if (typeof previousSingle === 'string' && previousSingle.trim().length > 0) { + merged[spec.configSingleField] = previousSingle; + } + } + } + if (!Object.hasOwn(input.next, 'enabled_tables') && Array.isArray(input.previous.enabled_tables)) { + merged.enabled_tables = input.previous.enabled_tables; + } + return merged; +} + function configuredScopeValues( connection: KtxProjectConnectionConfig | undefined, spec: ScopeDiscoverySpec, @@ -1156,18 +1294,19 @@ async function maybeConfigureSchemaScope(input: { prompts: KtxSetupDatabasesPromptAdapter; deps: KtxSetupDatabasesDeps; io: KtxCliIo; -}): Promise { + forcePrompt?: boolean; +}): Promise { const project = await loadKtxProject({ projectDir: input.projectDir }); const connection = project.config.connections[input.connectionId]; const driver = normalizeDriver(connection?.driver); - if (!driver) return true; + if (!driver) return 'ready'; const spec = SCOPE_DISCOVERY_SPECS[driver]; - if (!spec) return true; + if (!spec) return 'ready'; const arrayVal = connection?.[spec.configArrayField]; - if (Array.isArray(arrayVal) && arrayVal.length > 0) { - return true; + if (Array.isArray(arrayVal) && arrayVal.length > 0 && input.forcePrompt !== true) { + return 'ready'; } if (input.args.databaseSchemas.length > 0) { @@ -1177,7 +1316,7 @@ async function maybeConfigureSchemaScope(input: { values: input.args.databaseSchemas, spec, }); - return true; + return 'ready'; } writeSetupSection(input.io, `Discovering ${spec.promptLabel.toLowerCase()}`, [ @@ -1190,14 +1329,18 @@ async function maybeConfigureSchemaScope(input: { await (input.deps.listSchemas ?? defaultListSchemas)(input.projectDir, input.connectionId), ); } catch (error) { + const detail = error instanceof Error ? error.message : String(error); input.io.stderr.write( - `Could not discover ${spec.promptLabel.toLowerCase()} for ${input.connectionId}; continuing with existing ${spec.noun} scope. ` + - `Pass --database-schema to set it explicitly. ${error instanceof Error ? error.message : String(error)}\n`, + input.forcePrompt === true + ? `Could not discover ${spec.promptLabel.toLowerCase()} for ${input.connectionId}; edit was not saved. ` + + `Pass --database-schema to set it explicitly. ${detail}\n` + : `Could not discover ${spec.promptLabel.toLowerCase()} for ${input.connectionId}; continuing with existing ${spec.noun} scope. ` + + `Pass --database-schema to set it explicitly. ${detail}\n`, ); - return true; + return input.forcePrompt === true ? 'failed' : 'ready'; } if (discovered.length === 0) { - return true; + return 'ready'; } let selected: string[]; @@ -1217,7 +1360,7 @@ async function maybeConfigureSchemaScope(input: { required: true, }); if (choices.includes('back')) { - return false; + return 'back'; } selected = choices.length > 0 ? choices : initialValues; } @@ -1232,7 +1375,7 @@ async function maybeConfigureSchemaScope(input: { writeSetupSection(input.io, `${capitalNounPlural} saved for ${input.connectionId}`, [ `✓ ${selected.join(', ')}`, ]); - return true; + return 'ready'; } async function maybeConfigureTableScope(input: { @@ -1242,19 +1385,20 @@ async function maybeConfigureTableScope(input: { prompts: KtxSetupDatabasesPromptAdapter; io: KtxCliIo; deps: KtxSetupDatabasesDeps; -}): Promise { + forcePrompt?: boolean; +}): Promise { const project = await loadKtxProject({ projectDir: input.projectDir }); const connection = project.config.connections[input.connectionId]; const driver = normalizeDriver(connection?.driver); - if (!driver || driver === 'sqlite') return true; + if (!driver || driver === 'sqlite') return 'ready'; const existingTables = connection?.enabled_tables; - if (Array.isArray(existingTables) && existingTables.length > 0) { - return true; + if (Array.isArray(existingTables) && existingTables.length > 0 && input.forcePrompt !== true) { + return 'ready'; } if (input.args.inputMode === 'disabled') { - return true; + return 'ready'; } writeSetupSection(input.io, 'Discovering tables', [ @@ -1268,15 +1412,20 @@ async function maybeConfigureTableScope(input: { input.connectionId, ); } catch (error) { + const detail = error instanceof Error ? error.message : String(error); input.io.stderr.write( - `Could not discover tables for ${input.connectionId}; continuing without table filter. ` + - `${error instanceof Error ? error.message : String(error)}\n`, + input.forcePrompt === true + ? `Could not discover tables for ${input.connectionId}; edit was not saved. ${detail}\n` + : `Could not discover tables for ${input.connectionId}; continuing without table filter. ${detail}\n`, ); - return true; + return input.forcePrompt === true ? 'failed' : 'ready'; } if (discovered.length === 0) { - return true; + if (input.forcePrompt === true) { + input.io.stderr.write(`No tables discovered for ${input.connectionId}; edit was not saved.\n`); + } + return input.forcePrompt === true ? 'failed' : 'ready'; } const allQualified = discovered.map((t) => `${t.schema}.${t.name}`); @@ -1290,7 +1439,7 @@ async function maybeConfigureTableScope(input: { writeSetupSection(input.io, `Tables enabled for ${input.connectionId}`, [ `✓ ${allQualified[0]}`, ]); - return true; + return 'ready'; } const bySchema = new Map(); @@ -1316,7 +1465,7 @@ async function maybeConfigureTableScope(input: { }); if (action === 'back') { - return false; + return 'back'; } if (action === 'all') { @@ -1332,7 +1481,10 @@ async function maybeConfigureTableScope(input: { const suffix = t.kind === 'view' ? ' (view)' : ''; return { value: qualified, label: `${qualified}${suffix}` }; }), - initialValues: allQualified, + initialValues: + Array.isArray(existingTables) && input.forcePrompt === true + ? existingTables.filter((table): table is string => typeof table === 'string' && allQualified.includes(table)) + : allQualified, required: true, }); @@ -1356,7 +1508,7 @@ async function maybeConfigureTableScope(input: { writeSetupSection(input.io, `Tables enabled for ${input.connectionId}`, [ `✓ ${selected.length}/${discovered.length} tables enabled`, ]); - return true; + return 'ready'; } async function ensureHistoricSqlIngestDefaults(projectDir: string): Promise { @@ -1466,7 +1618,8 @@ async function validateAndScanConnection(input: { deps: KtxSetupDatabasesDeps; args: KtxSetupDatabasesArgs; prompts: KtxSetupDatabasesPromptAdapter; -}): Promise { + forceScopeAndTables?: boolean; +}): Promise { const testConnection = input.deps.testConnection ?? defaultTestConnection; const scanConnection = input.deps.scanConnection ?? defaultScanConnection; const project = await loadKtxProject({ projectDir: input.projectDir }); @@ -1477,7 +1630,7 @@ async function validateAndScanConnection(input: { if (testCode !== 0) { flushBufferedCommandOutput(input.io, testIo); input.io.stderr.write(`Connection test failed for ${input.connectionId}.\n`); - return false; + return 'failed'; } const testOutput = testIo.stdoutText(); const outputDriver = normalizeDriver(readOutputValue(testOutput, 'Driver')); @@ -1486,14 +1639,24 @@ async function validateAndScanConnection(input: { writeSetupSection(input.io, `Testing ${input.connectionId}`, testLines); while (true) { - if (!(await maybeConfigureSchemaScope(input))) { - return false; + const schemaStatus = await maybeConfigureSchemaScope({ ...input, forcePrompt: input.forceScopeAndTables }); + if (schemaStatus !== 'ready') { + return schemaStatus; } - if (await maybeConfigureTableScope(input)) { + const tableStatus = await maybeConfigureTableScope({ ...input, forcePrompt: input.forceScopeAndTables }); + if (tableStatus === 'ready') { break; } + if (input.forceScopeAndTables) { + return tableStatus; + } + + if (tableStatus === 'failed') { + return 'failed'; + } + await clearScopeConfig(input.projectDir, input.connectionId); } @@ -1554,7 +1717,7 @@ async function validateAndScanConnection(input: { ); } if (scanCode !== 0) { - return false; + return 'failed'; } } const scanOutput = scanIo.stdoutText(); @@ -1570,14 +1733,14 @@ async function validateAndScanConnection(input: { writeSetupSection(input.io, 'Primary source ready', [ `${input.connectionId} · ${driverDisplay} · structural scan complete`, ]); - return true; + return 'ready'; } async function chooseDrivers( args: KtxSetupDatabasesArgs, io: KtxCliIo, prompts: KtxSetupDatabasesPromptAdapter, - options?: { hasPrimarySources?: boolean }, + options?: { hasPrimarySources?: boolean; initialDrivers?: KtxSetupDatabaseDriver[] }, ): Promise { if (args.databaseDrivers && args.databaseDrivers.length > 0) { return [...new Set(args.databaseDrivers)]; @@ -1592,10 +1755,12 @@ async function chooseDrivers( return 'missing-input'; } while (true) { + const initialValues = unique(options?.initialDrivers ?? []); const choices = await prompts.multiselect({ message: withMultiselectNavigation('Which primary sources should KTX connect to?'), options: [...DRIVER_OPTIONS], - required: false, + ...(initialValues.length > 0 ? { initialValues } : {}), + required: options?.hasPrimarySources === true, }); if (choices.includes('back')) { return 'back'; @@ -1617,7 +1782,7 @@ async function chooseConnectionIdForDriver(input: { connections: Record; args: KtxSetupDatabasesArgs; prompts: KtxSetupDatabasesPromptAdapter; -}): Promise<{ kind: 'existing' | 'new'; connectionId: string } | 'back' | 'missing-input'> { +}): Promise<{ kind: 'existing' | 'new' | 'edit'; connectionId: string } | 'back' | 'missing-input'> { if (input.args.databaseConnectionId) { return { kind: 'new', connectionId: input.args.databaseConnectionId }; } @@ -1647,14 +1812,19 @@ async function chooseConnectionIdForDriver(input: { options: [ ...existingIds.map((connectionId) => ({ value: `existing:${connectionId}`, - label: `Use existing ${label} connection: ${connectionId}`, + label: `Keep existing ${label} connection: ${connectionId}`, })), - { value: 'new', label: `Add new ${label} connection` }, + ...existingIds.map((connectionId) => ({ + value: `edit:${connectionId}`, + label: `Edit ${label} connection: ${connectionId}`, + })), + { value: 'new', label: `Add another ${label} connection` }, { value: 'back', label: 'Back' }, ], }); if (choice === 'back') return 'back'; if (choice.startsWith('existing:')) return { kind: 'existing', connectionId: choice.slice('existing:'.length) }; + if (choice.startsWith('edit:')) return { kind: 'edit', connectionId: choice.slice('edit:'.length) }; const entered = await input.prompts.text({ message: withTextInputNavigation(connectionNamePrompt(label)), placeholder: defaultId, @@ -1666,6 +1836,102 @@ async function chooseConnectionIdForDriver(input: { } } +async function choosePrimarySourceToEdit(input: { + projectDir: string; + connectionIds: string[]; + prompts: KtxSetupDatabasesPromptAdapter; +}): Promise { + const project = await loadKtxProject({ projectDir: input.projectDir }); + const options = input.connectionIds + .map((connectionId) => { + const driver = normalizeDriver(project.config.connections[connectionId]?.driver); + if (!driver) return null; + return { value: connectionId, label: `${connectionId} (${driverLabel(driver)})` }; + }) + .filter((option): option is { value: string; label: string } => option !== null); + if (options.length === 0) return 'back'; + const choice = await input.prompts.select({ + message: 'Primary source to edit', + options: [...options, { value: 'back', label: 'Back' }], + }); + return choice === 'back' ? 'back' : choice; +} + +async function runPrimarySourceFullEdit(input: { + projectDir: string; + connectionId: string; + args: KtxSetupDatabasesArgs; + prompts: KtxSetupDatabasesPromptAdapter; + io: KtxCliIo; + deps: KtxSetupDatabasesDeps; +}): Promise<'ready' | 'back' | 'failed'> { + const project = await loadKtxProject({ projectDir: input.projectDir }); + const existing = project.config.connections[input.connectionId]; + const driver = normalizeDriver(existing?.driver); + if (!existing || !driver) { + input.io.stderr.write(`Connection "${input.connectionId}" is not a configured primary source.\n`); + return 'failed'; + } + + const rollback = await createConnectionConfigRollback(input.projectDir, input.connectionId); + const replacement = await buildConnectionConfig({ + driver, + connectionId: input.connectionId, + args: input.args, + prompts: input.prompts, + existingConnection: existing, + }); + if (replacement === 'back') { + await rollback(); + return 'back'; + } + if (!replacement) { + await rollback(); + return 'failed'; + } + + const withHistoricSql = await maybeApplyHistoricSqlConfig({ + connection: replacement, + driver, + args: input.args, + prompts: input.prompts, + }); + if (withHistoricSql === 'back') { + await rollback(); + return 'back'; + } + + await writeConnectionConfig({ + projectDir: input.projectDir, + connectionId: input.connectionId, + connection: withExistingPrimaryEditPromptDefaults({ + previous: existing, + next: { + ...withHistoricSql, + ...(!Object.hasOwn(withHistoricSql, 'historicSql') && existing.historicSql !== undefined + ? { historicSql: existing.historicSql } + : {}), + }, + driver, + }), + }); + + const validated = await validateAndScanConnection({ + projectDir: input.projectDir, + connectionId: input.connectionId, + io: input.io, + deps: input.deps, + args: input.args, + prompts: input.prompts, + forceScopeAndTables: true, + }); + if (validated !== 'ready') { + await rollback(); + return validated; + } + return 'ready'; +} + export async function runKtxSetupDatabasesStep( args: KtxSetupDatabasesArgs, io: KtxCliIo, @@ -1688,7 +1954,18 @@ export async function runKtxSetupDatabasesStep( prompts, }); if (historicSqlResult === 'back') return { status: 'back', projectDir: args.projectDir }; - if (!(await validateAndScanConnection({ projectDir: args.projectDir, connectionId, io, deps, args, prompts }))) { + const setupStatus = await validateAndScanConnection({ + projectDir: args.projectDir, + connectionId, + io, + deps, + args, + prompts, + }); + if (setupStatus === 'back') { + return { status: 'back', projectDir: args.projectDir }; + } + if (setupStatus === 'failed') { return { status: 'failed', projectDir: args.projectDir }; } selectedConnectionIds.push(connectionId); @@ -1712,10 +1989,43 @@ export async function runKtxSetupDatabasesStep( await markDatabasesComplete(args.projectDir, selectedConnectionIds); return { status: 'ready', projectDir: args.projectDir, connectionIds: selectedConnectionIds }; } + if (action === 'edit') { + const connectionId = await choosePrimarySourceToEdit({ + projectDir: args.projectDir, + connectionIds: selectedConnectionIds, + prompts, + }); + if (connectionId === 'back') { + showConfiguredPrimaryMenu = true; + continue; + } + const editResult = await runPrimarySourceFullEdit({ + projectDir: args.projectDir, + connectionId, + args, + prompts, + io, + deps, + }); + if (editResult === 'back') { + showConfiguredPrimaryMenu = true; + continue; + } + if (editResult === 'failed') { + return { status: 'failed', projectDir: args.projectDir }; + } + pushUniqueConnectionId(selectedConnectionIds, connectionId); + showConfiguredPrimaryMenu = true; + continue; + } } showConfiguredPrimaryMenu = false; - const drivers = await chooseDrivers(args, io, prompts, { hasPrimarySources: selectedConnectionIds.length > 0 }); + const driverProject = await loadKtxProject({ projectDir: args.projectDir }); + const drivers = await chooseDrivers(args, io, prompts, { + hasPrimarySources: selectedConnectionIds.length > 0, + initialDrivers: configuredPrimaryDrivers(driverProject.config.connections, selectedConnectionIds), + }); if (drivers === 'back') { if (selectedConnectionIds.length > 0 && canReturnToDriverSelection && args.inputMode !== 'disabled') { showConfiguredPrimaryMenu = true; @@ -1750,7 +2060,26 @@ export async function runKtxSetupDatabasesStep( return { status: 'missing-input', projectDir: args.projectDir }; } - if (connectionChoice.kind === 'new') { + let connectionAlreadyValidated = false; + if (connectionChoice.kind === 'edit') { + const editResult = await runPrimarySourceFullEdit({ + projectDir: args.projectDir, + connectionId: connectionChoice.connectionId, + args, + prompts, + io, + deps, + }); + if (editResult === 'back') { + if (!canReturnToDriverSelection) return { status: 'back', projectDir: args.projectDir }; + returnToDriverSelection = true; + break; + } + if (editResult === 'failed') { + return { status: 'failed', projectDir: args.projectDir }; + } + connectionAlreadyValidated = true; + } else if (connectionChoice.kind === 'new') { let connection = await buildConnectionConfig({ driver, connectionId: connectionChoice.connectionId, @@ -1819,16 +2148,22 @@ export async function runKtxSetupDatabasesStep( } let connectionSkipped = false; - while ( - !(await validateAndScanConnection({ - projectDir: args.projectDir, - connectionId: connectionChoice.connectionId, - io, - deps, - args, - prompts, - })) - ) { + let setupStatus: ConnectionSetupStatus = connectionAlreadyValidated + ? 'ready' + : await validateAndScanConnection({ + projectDir: args.projectDir, + connectionId: connectionChoice.connectionId, + io, + deps, + args, + prompts, + }); + while (!connectionAlreadyValidated && setupStatus !== 'ready') { + if (setupStatus === 'back') { + if (!canReturnToDriverSelection) return { status: 'back', projectDir: args.projectDir }; + returnToDriverSelection = true; + break; + } if (args.inputMode === 'disabled') return { status: 'failed', projectDir: args.projectDir }; const action = await prompts.select({ message: `Primary source setup failed for ${connectionChoice.connectionId}`, @@ -1848,7 +2183,16 @@ export async function runKtxSetupDatabasesStep( connectionSkipped = true; break; } - if (action === 're-enter') { + if (action === 'retry') { + setupStatus = await validateAndScanConnection({ + projectDir: args.projectDir, + connectionId: connectionChoice.connectionId, + io, + deps, + args, + prompts, + }); + } else if (action === 're-enter') { const connection = await buildConnectionConfig({ driver, connectionId: connectionChoice.connectionId, @@ -1872,6 +2216,14 @@ export async function runKtxSetupDatabasesStep( connectionId: connectionChoice.connectionId, connection: withHistoricSql, }); + setupStatus = await validateAndScanConnection({ + projectDir: args.projectDir, + connectionId: connectionChoice.connectionId, + io, + deps, + args, + prompts, + }); } } if (returnToDriverSelection) break; diff --git a/packages/cli/src/setup-sources.test.ts b/packages/cli/src/setup-sources.test.ts index 7fe61f76..0a0eab2c 100644 --- a/packages/cli/src/setup-sources.test.ts +++ b/packages/cli/src/setup-sources.test.ts @@ -861,6 +861,7 @@ describe('setup sources step', () => { message: 'Configure dbt', options: [ { value: 'existing:dbt-main', label: 'Use existing dbt connection: dbt-main' }, + { value: 'edit:dbt-main', label: 'Edit existing dbt connection: dbt-main' }, { value: 'new', label: 'Add new dbt connection' }, { value: 'back', label: 'Back' }, ], @@ -988,6 +989,10 @@ describe('setup sources step', () => { value: `existing:${testCase.connectionId}`, label: `Use existing ${testCase.expectedLabel} connection: ${testCase.connectionId}`, }, + { + value: `edit:${testCase.connectionId}`, + label: `Edit existing ${testCase.expectedLabel} connection: ${testCase.connectionId}`, + }, { value: 'new', label: `Add new ${testCase.expectedLabel} connection` }, { value: 'back', label: 'Back' }, ], @@ -996,6 +1001,314 @@ describe('setup sources step', () => { } }); + it('edits an existing Notion source and reopens the page picker with stored pages selected', async () => { + await addPrimarySource(); + await addConnection('notion-main', { + driver: 'notion', + auth_token_ref: 'env:NOTION_TOKEN', + crawl_mode: 'selected_roots', + root_page_ids: ['old-page'], + root_database_ids: [], + root_data_source_ids: [], + }); + const validateNotion = vi.fn(async () => ({ ok: true as const, detail: 'roots=1' })); + const pickNotionRootPages = vi.fn(async () => ({ kind: 'selected' as const, rootPageIds: ['new-page'] })); + const testPrompts = prompts({ + multiselect: [['notion']], + select: ['edit:notion-main', 'keep', 'selected_roots', 'done'], + }); + + await expect( + runKtxSetupSourcesStep( + { projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false }, + makeIo().io, + { + prompts: testPrompts, + validateNotion, + pickNotionRootPages, + }, + ), + ).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['notion-main'] }); + + expect(testPrompts.select).toHaveBeenCalledWith({ + message: 'How should KTX find your Notion integration token?', + options: [ + { value: 'keep', label: 'Keep existing credential' }, + { value: 'env', label: 'Use NOTION_TOKEN from the environment' }, + { value: 'paste', label: 'Paste a key and save it as a local secret file' }, + { value: 'back', label: 'Back' }, + ], + }); + expect(pickNotionRootPages).toHaveBeenCalledWith( + { + connectionId: 'notion-main', + connection: expect.objectContaining({ + driver: 'notion', + auth_token_ref: 'env:NOTION_TOKEN', + crawl_mode: 'selected_roots', + root_page_ids: ['old-page'], + }), + }, + expect.anything(), + ); + expect((await readConfig()).connections['notion-main']).toMatchObject({ + driver: 'notion', + auth_token_ref: 'env:NOTION_TOKEN', + crawl_mode: 'selected_roots', + root_page_ids: ['new-page'], + }); + }); + + it('edits an existing Metabase source with the current URL and credential as defaults', async () => { + await addPrimarySource(); + await addConnection('metabase-main', { + driver: 'metabase', + api_url: 'https://metabase-old.example.com', + api_key_ref: 'env:METABASE_API_KEY', // pragma: allowlist secret + mappings: { + databaseMappings: { '1': 'warehouse' }, + syncEnabled: { '1': true }, + syncMode: 'ALL', + }, + }); + const testPrompts = prompts({ + multiselect: [['metabase']], + select: ['edit:metabase-main', 'keep', 'done'], + text: ['https://metabase-new.example.com'], + }); + const discoverMetabaseDatabases = vi.fn(async () => [ + { id: 2, name: 'Analytics', engine: 'postgres', host: 'db.example.com', dbName: 'analytics' }, + ]); + + await expect( + runKtxSetupSourcesStep( + { projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false }, + makeIo().io, + { + prompts: testPrompts, + discoverMetabaseDatabases, + validateMetabase: vi.fn(async () => ({ ok: true as const, detail: 'mapping validated' })), + runMapping: vi.fn(async () => 0), + }, + ), + ).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['metabase-main'] }); + + expect(testPrompts.text).toHaveBeenCalledWith({ + message: textInputPrompt('Metabase URL'), + initialValue: 'https://metabase-old.example.com', + }); + expect(testPrompts.select).toHaveBeenCalledWith({ + message: 'How should KTX find your Metabase API key?', + options: [ + { value: 'keep', label: 'Keep existing credential' }, + { value: 'env', label: 'Use METABASE_API_KEY from the environment' }, + { value: 'paste', label: 'Paste a key and save it as a local secret file' }, + { value: 'back', label: 'Back' }, + ], + }); + expect(discoverMetabaseDatabases).toHaveBeenCalledWith({ + sourceUrl: 'https://metabase-new.example.com', + sourceApiKeyRef: 'env:METABASE_API_KEY', + sourceConnectionId: 'metabase-main', + }); + expect((await readConfig()).connections['metabase-main']).toMatchObject({ + driver: 'metabase', + api_url: 'https://metabase-new.example.com', + api_key_ref: 'env:METABASE_API_KEY', + mappings: { + databaseMappings: { '2': 'warehouse' }, + syncEnabled: { '2': true }, + syncMode: 'ALL', + }, + }); + }); + + it('rolls back an edited context source when validation fails', async () => { + await addPrimarySource(); + await addConnection('dbt-main', { + driver: 'dbt', + source_dir: '/repo/existing-dbt', + project_name: 'analytics', + }); + const validateDbt = vi.fn(async () => ({ ok: false as const, message: 'dbt project not found' })); + const testPrompts = prompts({ + multiselect: [['dbt']], + select: ['edit:dbt-main', 'path'], + text: ['/repo/new-dbt', ''], + }); + + await expect( + runKtxSetupSourcesStep( + { projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false }, + makeIo().io, + { + prompts: testPrompts, + validateDbt, + }, + ), + ).resolves.toEqual({ status: 'failed', projectDir }); + + expect(validateDbt).toHaveBeenCalledWith(expect.objectContaining({ + driver: 'dbt', + source_dir: '/repo/new-dbt', + })); + const config = await readConfig(); + expect(config.connections['dbt-main']).toMatchObject({ + driver: 'dbt', + source_dir: '/repo/existing-dbt', + project_name: 'analytics', + }); + expect(config.ingest.adapters).not.toContain('dbt'); + }); + + it('lets git-backed context source edits keep the existing repo credential', async () => { + await addPrimarySource(); + await addConnection('metricflow-main', { + driver: 'metricflow', + metricflow: { + repoUrl: 'https://github.com/acme/private-metricflow', + branch: 'main', + path: 'metrics', + auth_token_ref: 'env:METRICFLOW_REPO_TOKEN', // pragma: allowlist secret + }, + }); + const testGitRepo = vi.fn(async () => ({ ok: false as const, error: 'authentication required' })); + const testPrompts = prompts({ + multiselect: [['metricflow']], + select: ['edit:metricflow-main', 'git', 'keep', 'done'], + text: ['https://github.com/acme/private-metricflow', 'main', 'metrics'], + }); + + await expect( + runKtxSetupSourcesStep( + { projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false }, + makeIo().io, + { + prompts: testPrompts, + testGitRepo, + validateMetricflow: vi.fn(async () => ({ ok: true as const, detail: 'metrics=1' })), + }, + ), + ).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['metricflow-main'] }); + + expect(testPrompts.select).toHaveBeenCalledWith({ + message: 'This MetricFlow repo requires authentication.', + options: [ + { value: 'keep', label: 'Keep existing credential' }, + { value: 'env', label: 'Use GITHUB_TOKEN from the environment' }, + { value: 'paste', label: 'Paste a token and save it as a local secret file' }, + { value: 'skip', label: 'Skip — try without authentication' }, + { value: 'back', label: 'Back' }, + ], + }); + expect((await readConfig()).connections['metricflow-main']).toMatchObject({ + driver: 'metricflow', + metricflow: { + repoUrl: 'https://github.com/acme/private-metricflow', + branch: 'main', + path: 'metrics', + auth_token_ref: 'env:METRICFLOW_REPO_TOKEN', + }, + }); + }); + + it('edits an existing context source from the configured-source follow-up menu', async () => { + await addPrimarySource(); + await addConnection('dbt-main', { + driver: 'dbt', + source_dir: '/repo/existing-dbt', + project_name: 'analytics', + }); + const validateDbt = vi.fn(async () => ({ ok: true as const, detail: 'project=analytics schemas=2' })); + const testPrompts = prompts({ + multiselect: [['dbt']], + select: ['existing:dbt-main', 'edit', 'dbt-main', 'path', 'done'], + text: ['/repo/edited-dbt', ''], + }); + + await expect( + runKtxSetupSourcesStep( + { projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false }, + makeIo().io, + { + prompts: testPrompts, + validateDbt, + }, + ), + ).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['dbt-main'] }); + + expect(testPrompts.select).toHaveBeenCalledWith({ + message: '1 context source configured (dbt-main). Add another?', + options: [ + { value: 'done', label: 'Done — continue to context build' }, + { value: 'edit', label: 'Edit an existing context source' }, + { value: 'add', label: 'Add another context source' }, + ], + }); + expect(testPrompts.select).toHaveBeenCalledWith({ + message: 'Context source to edit', + options: [ + { value: 'dbt-main', label: 'dbt-main (dbt)' }, + { value: 'back', label: 'Back' }, + ], + }); + expect(testPrompts.text).toHaveBeenCalledWith({ + message: textInputPrompt('dbt local path'), + initialValue: '/repo/existing-dbt', + }); + expect(validateDbt).toHaveBeenLastCalledWith(expect.objectContaining({ + driver: 'dbt', + source_dir: '/repo/edited-dbt', + })); + expect((await readConfig()).connections['dbt-main']).toMatchObject({ + driver: 'dbt', + source_dir: '/repo/edited-dbt', + project_name: 'analytics', + }); + }); + + it('backs out of editing an existing context source to the source connection menu', async () => { + await addPrimarySource(); + await addConnection('dbt-main', { + driver: 'dbt', + source_dir: '/repo/existing-dbt', + project_name: 'analytics', + }); + const validateDbt = vi.fn(async () => ({ ok: true as const, detail: 'project=analytics schemas=2' })); + const testPrompts = prompts({ + multiselect: [['dbt']], + select: ['edit:dbt-main', 'back', 'existing:dbt-main'], + }); + + await expect( + runKtxSetupSourcesStep( + { projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false }, + makeIo().io, + { + prompts: testPrompts, + validateDbt, + }, + ), + ).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['dbt-main'] }); + + expect( + vi + .mocked(testPrompts.select) + .mock.calls.map(([options]) => options.message) + .filter((message) => message === 'Configure dbt'), + ).toHaveLength(2); + expect(validateDbt).toHaveBeenCalledWith({ + driver: 'dbt', + source_dir: '/repo/existing-dbt', + project_name: 'analytics', + }); + expect((await readConfig()).connections['dbt-main']).toMatchObject({ + driver: 'dbt', + source_dir: '/repo/existing-dbt', + project_name: 'analytics', + }); + }); + it('lets Escape from dbt git URL return to source location selection', async () => { await addPrimarySource(); const validateDbt = vi.fn(async () => ({ ok: true as const, detail: 'project=analytics schemas=2' })); diff --git a/packages/cli/src/setup-sources.ts b/packages/cli/src/setup-sources.ts index d18004d9..e0819e4a 100644 --- a/packages/cli/src/setup-sources.ts +++ b/packages/cli/src/setup-sources.ts @@ -224,17 +224,20 @@ async function chooseSourceCredentialRef(input: { label: string; envName: string; secretFileName: string; + existingRef?: string; }): Promise { while (true) { const choice = await input.prompts.select({ message: `How should KTX find your ${input.label}?`, options: [ + ...(input.existingRef ? [{ value: 'keep', label: 'Keep existing credential' }] : []), { value: 'env', label: `Use ${input.envName} from the environment` }, { value: 'paste', label: 'Paste a key and save it as a local secret file' }, { value: 'back', label: 'Back' }, ], }); if (choice === 'back') return 'back'; + if (choice === 'keep' && input.existingRef) return input.existingRef; if (choice === 'paste') { const value = await input.prompts.password({ message: input.label }); if (value === undefined) continue; @@ -256,12 +259,14 @@ async function chooseGitAuthCredentialRef(input: { projectDir: string; source: KtxSetupSourceType; connectionId: string; + existingRef?: string; }): Promise { const label = input.source === 'dbt' ? 'This' : `This ${sourceLabel(input.source)}`; while (true) { const choice = await input.prompts.select({ message: `${label} repo requires authentication.`, options: [ + ...(input.existingRef ? [{ value: 'keep', label: 'Keep existing credential' }] : []), { value: 'env', label: 'Use GITHUB_TOKEN from the environment' }, { value: 'paste', label: 'Paste a token and save it as a local secret file' }, { value: 'skip', label: 'Skip — try without authentication' }, @@ -269,6 +274,7 @@ async function chooseGitAuthCredentialRef(input: { ], }); if (choice === 'back') return 'back'; + if (choice === 'keep' && input.existingRef) return input.existingRef; if (choice === 'skip') return undefined; if (choice === 'paste') { const value = await input.prompts.password({ message: 'Git access token' }); @@ -793,8 +799,14 @@ interface WarehouseConnectionChoice { type InteractiveSourceConnectionChoice = | { kind: 'existing'; connectionId: string; connection: KtxProjectConnectionConfig } | { kind: 'new'; args: KtxSetupSourcesArgs } + | { kind: 'edited'; connectionId: string; args: KtxSetupSourcesArgs } | 'back'; +type SourceSetupChoiceResult = + | { status: 'ready'; connectionId: string } + | { status: 'back' } + | { status: 'failed' }; + async function runSourcePromptSteps( initialState: SourcePromptState, stepsForState: (state: SourcePromptState) => SourcePromptStep[], @@ -828,6 +840,12 @@ function resetRepoLocationFields(state: SourcePromptState): void { delete state.sourceProjectName; } +function sourceLocationFromArgs(args: KtxSetupSourcesArgs): SourceLocationChoice | undefined { + if (args.sourcePath) return 'path'; + if (args.sourceGitUrl) return 'git'; + return undefined; +} + function warehouseConnectionChoices(config: KtxProjectConfig): WarehouseConnectionChoice[] { return Object.entries(config.connections) .filter(([, connection]) => PRIMARY_SOURCE_DRIVERS.has(String(connection.driver ?? '').toLowerCase())) @@ -964,7 +982,7 @@ async function promptForInteractiveSource( testGitRepo: KtxSetupSourcesDeps['testGitRepo'] = testRepoConnection, discoverMetabaseDatabaseList?: KtxSetupSourcesDeps['discoverMetabaseDatabases'], ): Promise { - const initialState: SourcePromptState = { ...args, source }; + const initialState: SourcePromptState = { ...args, source, sourceLocation: sourceLocationFromArgs(args) }; if (args.sourceConnectionId) { initialState.sourceConnectionId = args.sourceConnectionId; } @@ -994,7 +1012,10 @@ async function promptForInteractiveSource( ...(state.sourceLocation === 'path' ? [ async (currentState: SourcePromptState) => { - const sourcePath = await promptText(prompts, { message: `${source} local path` }); + const sourcePath = await promptText(prompts, { + message: `${source} local path`, + ...(currentState.sourcePath ? { initialValue: currentState.sourcePath } : {}), + }); if (sourcePath === undefined) return 'back'; currentState.sourcePath = sourcePath; return 'next'; @@ -1004,13 +1025,19 @@ async function promptForInteractiveSource( ...(state.sourceLocation === 'git' ? [ async (currentState: SourcePromptState) => { - const sourceGitUrl = await promptText(prompts, { message: `${source} git URL` }); + const sourceGitUrl = await promptText(prompts, { + message: `${source} git URL`, + ...(currentState.sourceGitUrl ? { initialValue: currentState.sourceGitUrl } : {}), + }); if (sourceGitUrl === undefined) return 'back'; currentState.sourceGitUrl = sourceGitUrl; return 'next'; }, async (currentState: SourcePromptState) => { - const branch = await promptText(prompts, { message: `${source} git branch`, initialValue: 'main' }); + const branch = await promptText(prompts, { + message: `${source} git branch`, + initialValue: currentState.sourceBranch ?? 'main', + }); if (branch === undefined) return 'back'; currentState.sourceBranch = branch || 'main'; return 'next'; @@ -1031,6 +1058,7 @@ async function promptForInteractiveSource( projectDir: args.projectDir, source, connectionId: currentState.sourceConnectionId ?? `${source}-main`, + existingRef: currentState.sourceAuthTokenRef, }); if (authRef === 'back') return 'back'; if (authRef) { @@ -1104,6 +1132,7 @@ async function promptForInteractiveSource( const subpath = await promptText(prompts, { message: sourceSubpathPrompt(source), placeholder: 'optional', + ...(currentState.sourceSubpath ? { initialValue: currentState.sourceSubpath } : {}), }); if (subpath === undefined) return 'back'; if (subpath) { @@ -1122,7 +1151,10 @@ async function promptForInteractiveSource( return await runSourcePromptSteps(initialState, () => [ ...connectionSteps, async (state) => { - const sourceUrl = await promptText(prompts, { message: 'Metabase URL' }); + const sourceUrl = await promptText(prompts, { + message: 'Metabase URL', + ...(state.sourceUrl ? { initialValue: state.sourceUrl } : {}), + }); if (sourceUrl === undefined) return 'back'; state.sourceUrl = sourceUrl; return 'next'; @@ -1134,6 +1166,7 @@ async function promptForInteractiveSource( label: 'Metabase API key', envName: 'METABASE_API_KEY', secretFileName: `${state.sourceConnectionId ?? 'metabase-main'}-api-key`, + existingRef: state.sourceApiKeyRef, }); if (ref === 'back') return 'back'; state.sourceApiKeyRef = ref; @@ -1165,13 +1198,19 @@ async function promptForInteractiveSource( return await runSourcePromptSteps(initialState, () => [ ...connectionSteps, async (state) => { - const sourceUrl = await promptText(prompts, { message: 'Looker base URL' }); + const sourceUrl = await promptText(prompts, { + message: 'Looker base URL', + ...(state.sourceUrl ? { initialValue: state.sourceUrl } : {}), + }); if (sourceUrl === undefined) return 'back'; state.sourceUrl = sourceUrl; return 'next'; }, async (state) => { - const sourceClientId = await promptText(prompts, { message: 'Looker client id' }); + const sourceClientId = await promptText(prompts, { + message: 'Looker client id', + ...(state.sourceClientId ? { initialValue: state.sourceClientId } : {}), + }); if (sourceClientId === undefined) return 'back'; state.sourceClientId = sourceClientId; return 'next'; @@ -1183,6 +1222,7 @@ async function promptForInteractiveSource( label: 'Looker client secret', envName: 'LOOKER_CLIENT_SECRET', secretFileName: `${state.sourceConnectionId ?? 'looker-main'}-client-secret`, + existingRef: state.sourceClientSecretRef, }); if (ref === 'back') return 'back'; state.sourceClientSecretRef = ref; @@ -1201,6 +1241,7 @@ async function promptForInteractiveSource( const lookerConnectionName = await promptText(prompts, { message: 'Looker connection name', placeholder: 'optional', + ...(state.sourceTarget ? { initialValue: state.sourceTarget } : {}), }); if (lookerConnectionName === undefined) return 'back'; if (lookerConnectionName) { @@ -1222,6 +1263,7 @@ async function promptForInteractiveSource( label: 'Notion integration token', envName: 'NOTION_TOKEN', secretFileName: `${currentState.sourceConnectionId ?? 'notion-main'}-token`, + existingRef: currentState.sourceApiKeyRef, }); if (ref === 'back') return 'back'; currentState.sourceApiKeyRef = ref; @@ -1286,6 +1328,24 @@ function existingConnectionIdsBySource( .sort((left, right) => left.localeCompare(right)); } +function sourceTypeForConnection(connection: KtxProjectConnectionConfig): KtxSetupSourceType | null { + const driver = String(connection.driver ?? '').toLowerCase(); + return SOURCE_OPTIONS.some((option) => option.value === driver) ? (driver as KtxSetupSourceType) : null; +} + +function contextSourceEditTargets(connections: Record): Array<{ + connectionId: string; + source: KtxSetupSourceType; +}> { + return Object.entries(connections) + .map(([connectionId, connection]) => { + const source = sourceTypeForConnection(connection); + return source ? { connectionId, source } : null; + }) + .filter((target): target is { connectionId: string; source: KtxSetupSourceType } => target !== null) + .sort((left, right) => left.connectionId.localeCompare(right.connectionId)); +} + function sourceChecklistForConnections(connections: Record): { options: Array<{ value: KtxSetupSourceType; label: string; hint?: string }>; initialValues: KtxSetupSourceType[]; @@ -1317,6 +1377,180 @@ function defaultConnectionIdForSource( return `${base}-${index}`; } +function firstStringRecordEntry(value: unknown): [string, string] | undefined { + if (!isRecord(value)) return undefined; + for (const [key, raw] of Object.entries(value)) { + if (typeof raw === 'string' && raw.trim().length > 0) { + return [key, raw.trim()]; + } + } + return undefined; +} + +function applyRepoSourceArgs( + args: KtxSetupSourcesArgs, + input: { repoUrl?: string; sourceDir?: string; branch?: string; subpath?: string; authTokenRef?: string }, +): void { + if (input.sourceDir) { + args.sourcePath = input.sourceDir; + } else if (input.repoUrl?.startsWith('file:')) { + args.sourcePath = fileURLToPath(input.repoUrl); + } else if (input.repoUrl) { + args.sourceGitUrl = input.repoUrl; + } + if (input.branch) args.sourceBranch = input.branch; + if (input.subpath) args.sourceSubpath = input.subpath; + if (input.authTokenRef) args.sourceAuthTokenRef = input.authTokenRef; +} + +function sourceArgsFromExistingConnection(input: { + args: KtxSetupSourcesArgs; + source: KtxSetupSourceType; + connectionId: string; + connection: KtxProjectConnectionConfig; +}): KtxSetupSourcesArgs { + const sourceArgs: KtxSetupSourcesArgs = { + projectDir: input.args.projectDir, + inputMode: input.args.inputMode, + source: input.source, + sourceConnectionId: input.connectionId, + runInitialSourceIngest: input.args.runInitialSourceIngest, + skipSources: input.args.skipSources, + }; + + if (input.source === 'dbt') { + applyRepoSourceArgs(sourceArgs, { + sourceDir: stringField(input.connection.source_dir), + repoUrl: stringField(input.connection.repo_url), + branch: stringField(input.connection.branch), + subpath: stringField(input.connection.path), + authTokenRef: stringField(input.connection.auth_token_ref), + }); + const profilesPath = stringField(input.connection.profiles_path); + const target = stringField(input.connection.target); + const projectName = stringField(input.connection.project_name); + if (profilesPath) sourceArgs.sourceProfilesPath = profilesPath; + if (target) sourceArgs.sourceTarget = target; + if (projectName) sourceArgs.sourceProjectName = projectName; + return sourceArgs; + } + + if (input.source === 'metricflow') { + const metricflow = isRecord(input.connection.metricflow) ? input.connection.metricflow : {}; + applyRepoSourceArgs(sourceArgs, { + repoUrl: stringField(metricflow.repoUrl), + branch: stringField(metricflow.branch), + subpath: stringField(metricflow.path), + authTokenRef: stringField(metricflow.auth_token_ref), + }); + return sourceArgs; + } + + if (input.source === 'lookml') { + applyRepoSourceArgs(sourceArgs, { + repoUrl: stringField(input.connection.repoUrl), + branch: stringField(input.connection.branch), + subpath: stringField(input.connection.path), + authTokenRef: stringField(input.connection.auth_token_ref), + }); + const mappings = isRecord(input.connection.mappings) ? input.connection.mappings : {}; + const expectedLookerConnectionName = stringField(mappings.expectedLookerConnectionName); + if (expectedLookerConnectionName) sourceArgs.sourceTarget = expectedLookerConnectionName; + return sourceArgs; + } + + if (input.source === 'metabase') { + sourceArgs.sourceUrl = stringField(input.connection.api_url); + sourceArgs.sourceApiKeyRef = stringField(input.connection.api_key_ref); + const mappings = isRecord(input.connection.mappings) ? input.connection.mappings : {}; + const databaseMapping = firstStringRecordEntry(mappings.databaseMappings); + if (databaseMapping) { + sourceArgs.metabaseDatabaseId = Number.parseInt(databaseMapping[0], 10); + sourceArgs.sourceWarehouseConnectionId = databaseMapping[1]; + } + return sourceArgs; + } + + if (input.source === 'looker') { + sourceArgs.sourceUrl = stringField(input.connection.base_url); + sourceArgs.sourceClientId = stringField(input.connection.client_id); + sourceArgs.sourceClientSecretRef = stringField(input.connection.client_secret_ref); + const mappings = isRecord(input.connection.mappings) ? input.connection.mappings : {}; + const connectionMapping = firstStringRecordEntry(mappings.connectionMappings); + if (connectionMapping) { + sourceArgs.sourceTarget = connectionMapping[0]; + sourceArgs.sourceWarehouseConnectionId = connectionMapping[1]; + } + return sourceArgs; + } + + sourceArgs.sourceApiKeyRef = stringField(input.connection.auth_token_ref); + sourceArgs.notionCrawlMode = + input.connection.crawl_mode === 'all_accessible' ? 'all_accessible' : 'selected_roots'; + if (Array.isArray(input.connection.root_page_ids)) { + sourceArgs.notionRootPageIds = input.connection.root_page_ids.filter( + (pageId): pageId is string => typeof pageId === 'string', + ); + } + return sourceArgs; +} + +async function promptEditedSourceConnection(input: { + args: KtxSetupSourcesArgs; + source: KtxSetupSourceType; + connectionId: string; + connection: KtxProjectConnectionConfig; + prompts: KtxSetupSourcesPromptAdapter; + io: KtxCliIo; + testGitRepo?: KtxSetupSourcesDeps['testGitRepo']; + pickNotionRootPages?: KtxSetupSourcesDeps['pickNotionRootPages']; + discoverMetabaseDatabases?: KtxSetupSourcesDeps['discoverMetabaseDatabases']; +}): Promise | 'back'> { + const sourceArgs = await promptForInteractiveSource( + sourceArgsFromExistingConnection({ + args: input.args, + source: input.source, + connectionId: input.connectionId, + connection: input.connection, + }), + input.source, + input.prompts, + input.io, + { + pickNotionRootPages: input.pickNotionRootPages, + discoverMetabaseDatabases: input.discoverMetabaseDatabases, + }, + input.connectionId, + input.testGitRepo, + input.discoverMetabaseDatabases, + ); + return sourceArgs === 'back' + ? 'back' + : { kind: 'edited', connectionId: input.connectionId, args: sourceArgs }; +} + +async function chooseContextSourceToEdit(input: { + projectDir: string; + prompts: KtxSetupSourcesPromptAdapter; +}): Promise<{ connectionId: string; source: KtxSetupSourceType } | 'back'> { + const project = await loadKtxProject({ projectDir: input.projectDir }); + const targets = contextSourceEditTargets(project.config.connections); + if (targets.length === 0) return 'back'; + const choice = await input.prompts.select({ + message: 'Context source to edit', + options: [ + ...targets.map((target) => ({ + value: target.connectionId, + label: `${target.connectionId} (${sourceLabel(target.source)})`, + })), + { value: 'back', label: 'Back' }, + ], + }); + if (choice === 'back') return 'back'; + const target = targets.find((candidate) => candidate.connectionId === choice); + return target ?? 'back'; +} + async function chooseInteractiveSourceConnection(input: { args: KtxSetupSourcesArgs; source: KtxSetupSourceType; @@ -1356,6 +1590,10 @@ async function chooseInteractiveSourceConnection(input: { value: `existing:${connectionId}`, label: `Use existing ${label} connection: ${connectionId}`, })), + ...existingIds.map((connectionId) => ({ + value: `edit:${connectionId}`, + label: `Edit existing ${label} connection: ${connectionId}`, + })), { value: 'new', label: `Add new ${label} connection` }, { value: 'back', label: 'Back' }, ], @@ -1369,6 +1607,28 @@ async function chooseInteractiveSourceConnection(input: { } continue; } + if (choice.startsWith('edit:')) { + const connectionId = choice.slice('edit:'.length); + const connection = input.connections[connectionId]; + if (!connection) { + continue; + } + const edited = await promptEditedSourceConnection({ + args: input.args, + source: input.source, + connectionId, + connection, + prompts: input.prompts, + io: input.io, + testGitRepo: input.testGitRepo, + pickNotionRootPages: input.pickNotionRootPages, + discoverMetabaseDatabases: input.discoverMetabaseDatabases, + }); + if (edited === 'back') { + continue; + } + return edited; + } const sourceArgs = await promptForInteractiveSource( input.args, input.source, @@ -1433,6 +1693,85 @@ async function validateSource( return await (deps.validateNotion ?? defaultValidateNotion)(args.connection); } +async function saveValidateAndMaybeBuildSource(input: { + args: KtxSetupSourcesArgs; + source: KtxSetupSourceType; + sourceChoice: Exclude; + prompts: KtxSetupSourcesPromptAdapter; + io: KtxCliIo; + deps: KtxSetupSourcesDeps; +}): Promise { + const connectionId = + input.sourceChoice.kind === 'existing' + ? input.sourceChoice.connectionId + : input.sourceChoice.kind === 'edited' + ? input.sourceChoice.connectionId + : (input.sourceChoice.args.sourceConnectionId ?? `${input.source}-main`); + const connection = + input.sourceChoice.kind === 'existing' + ? input.sourceChoice.connection + : buildConnection(input.source, input.sourceChoice.args); + const rollback = + input.sourceChoice.kind === 'existing' + ? undefined + : await writeSourceConnection( + input.args.projectDir, + connectionId, + connection, + sourceAdapter(input.source), + ); + + if (input.sourceChoice.kind === 'existing') { + await ensureSourceAdapterEnabled(input.args.projectDir, input.source); + } + + const validation = await validateSource( + input.source, + { projectDir: input.args.projectDir, connectionId, connection }, + input.deps, + ); + if (!validation.ok) { + await rollback?.(); + input.io.stderr.write(`${validation.message}\n`); + return { status: 'failed' }; + } + + if (input.source === 'metabase' || input.source === 'looker') { + input.prompts.log?.(`Validating ${sourceLabel(input.source)} mapping…`); + const mappingCode = await (input.deps.runMapping ?? defaultRunMapping)( + input.args.projectDir, + connectionId, + createSetupPrefixedIo(input.io), + ); + if (mappingCode !== 0) { + await rollback?.(); + return { status: 'failed' }; + } + } + + if (input.args.runInitialSourceIngest) { + const ingestResult = await runInitialSourceIngestWithRecovery({ + args: input.args, + connectionId, + io: input.io, + prompts: input.prompts, + deps: input.deps, + }); + if (ingestResult === 'failed') { + await rollback?.(); + return { status: 'failed' }; + } + if (ingestResult === 'back') { + await rollback?.(); + return { status: 'back' }; + } + } else { + input.io.stdout.write(`│ Context source ${connectionId} saved. It will be built during the context build step.\n`); + } + + return { status: 'ready', connectionId }; +} + export async function runKtxSetupSourcesStep( args: KtxSetupSourcesArgs, io: KtxCliIo, @@ -1510,62 +1849,27 @@ export async function runKtxSetupSourcesStep( returnToSourceSelection = true; break; } - const connectionId = - sourceChoice.kind === 'existing' - ? sourceChoice.connectionId - : (sourceChoice.args.sourceConnectionId ?? `${source}-main`); - const connection = - sourceChoice.kind === 'existing' ? sourceChoice.connection : buildConnection(source, sourceChoice.args); - const rollback = - sourceChoice.kind === 'existing' - ? undefined - : await writeSourceConnection(args.projectDir, connectionId, connection, sourceAdapter(source)); - if (sourceChoice.kind === 'existing') { - await ensureSourceAdapterEnabled(args.projectDir, source); - } - const validation = await validateSource(source, { projectDir: args.projectDir, connectionId, connection }, deps); - - if (!validation.ok) { - await rollback?.(); - io.stderr.write(`${validation.message}\n`); + const choiceResult = await saveValidateAndMaybeBuildSource({ + args, + source, + sourceChoice, + prompts, + io, + deps, + }); + if (choiceResult.status === 'failed') { return { status: 'failed', projectDir: args.projectDir }; } - if (source === 'metabase' || source === 'looker') { - prompts.log?.(`Validating ${sourceLabel(source)} mapping…`); - const mappingCode = await (deps.runMapping ?? defaultRunMapping)( - args.projectDir, - connectionId, - createSetupPrefixedIo(io), - ); - if (mappingCode !== 0) { - await rollback?.(); - return { status: 'failed', projectDir: args.projectDir }; + if (choiceResult.status === 'back') { + if (args.source) { + return { status: 'back', projectDir: args.projectDir }; } + returnToSourceSelection = true; + break; } - if (args.runInitialSourceIngest) { - const ingestResult = await runInitialSourceIngestWithRecovery({ - args, - connectionId, - io, - prompts, - deps, - }); - if (ingestResult === 'failed') { - await rollback?.(); - return { status: 'failed', projectDir: args.projectDir }; - } - if (ingestResult === 'back') { - await rollback?.(); - if (args.source) { - return { status: 'back', projectDir: args.projectDir }; - } - returnToSourceSelection = true; - break; - } - } else { - io.stdout.write(`│ Context source ${connectionId} saved. It will be built during the context build step.\n`); + if (!readyConnectionIds.includes(choiceResult.connectionId)) { + readyConnectionIds.push(choiceResult.connectionId); } - readyConnectionIds.push(connectionId); } if (returnToSourceSelection) { @@ -1573,14 +1877,66 @@ export async function runKtxSetupSourcesStep( } if (readyConnectionIds.length > 0 && !args.source && args.inputMode !== 'disabled') { - const addMore = await prompts.select({ - message: `${readyConnectionIds.length} context source${readyConnectionIds.length > 1 ? 's' : ''} configured (${readyConnectionIds.join(', ')}). Add another?`, - options: [ - { value: 'done', label: 'Done — continue to context build' }, - { value: 'add', label: 'Add another context source' }, - ], - }); - if (addMore === 'add') { + let restartSourceSelection = false; + while (true) { + const addMore = await prompts.select({ + message: `${readyConnectionIds.length} context source${readyConnectionIds.length > 1 ? 's' : ''} configured (${readyConnectionIds.join(', ')}). Add another?`, + options: [ + { value: 'done', label: 'Done — continue to context build' }, + { value: 'edit', label: 'Edit an existing context source' }, + { value: 'add', label: 'Add another context source' }, + ], + }); + if (addMore === 'add') { + restartSourceSelection = true; + break; + } + if (addMore === 'edit') { + const editTarget = await chooseContextSourceToEdit({ projectDir: args.projectDir, prompts }); + if (editTarget === 'back') { + continue; + } + const projectForEdit = await loadKtxProject({ projectDir: args.projectDir }); + const connection = projectForEdit.config.connections[editTarget.connectionId]; + if (!connection) { + continue; + } + const sourceChoice = await promptEditedSourceConnection({ + args, + source: editTarget.source, + connectionId: editTarget.connectionId, + connection, + prompts, + io, + testGitRepo: deps.testGitRepo, + pickNotionRootPages: deps.pickNotionRootPages, + discoverMetabaseDatabases: deps.discoverMetabaseDatabases, + }); + if (sourceChoice === 'back') { + continue; + } + const choiceResult = await saveValidateAndMaybeBuildSource({ + args, + source: editTarget.source, + sourceChoice, + prompts, + io, + deps, + }); + if (choiceResult.status === 'failed') { + return { status: 'failed', projectDir: args.projectDir }; + } + if (choiceResult.status === 'back') { + continue; + } + if (!readyConnectionIds.includes(choiceResult.connectionId)) { + readyConnectionIds.push(choiceResult.connectionId); + } + continue; + } + break; + } + if (restartSourceSelection) { continue; } } From f219ba22a61e68fa1bcda3dfeb921955aba36aa4 Mon Sep 17 00:00:00 2001 From: Luca Martial <48870843+luca-martial@users.noreply.github.com> Date: Wed, 13 May 2026 17:28:48 -0400 Subject: [PATCH 4/7] feat(cli): redesign Notion page picker UI and add skip-empty flow (#78) Rework the inline picker to use a cleaner visual style (filled/empty square glyphs, bordered layout, clack-style header) and streamlined keybindings (Enter to confirm, Escape to quit, Right Arrow to expand). Replace the transient "select at least one" hint with a skip-empty confirmation prompt that exits cleanly via quit-without-save. Co-authored-by: Claude Opus 4.6 (1M context) --- .../cli/src/notion-page-picker-tree.test.ts | 25 ++--- packages/cli/src/notion-page-picker-tree.ts | 15 +-- .../cli/src/notion-page-picker-tui.test.tsx | 70 ++++++------ packages/cli/src/notion-page-picker-tui.tsx | 102 ++++++++++++------ 4 files changed, 126 insertions(+), 86 deletions(-) diff --git a/packages/cli/src/notion-page-picker-tree.test.ts b/packages/cli/src/notion-page-picker-tree.test.ts index 94b46b57..58e8c7ca 100644 --- a/packages/cli/src/notion-page-picker-tree.test.ts +++ b/packages/cli/src/notion-page-picker-tree.test.ts @@ -11,7 +11,6 @@ import { selectAllVisible, selectNone, toggleChecked, - TRANSIENT_HINT_DURATION_MS, visibleNodeIds, type NotionPickerPageInput, } from './notion-page-picker-tree.js'; @@ -223,22 +222,24 @@ describe('bulk actions and reducer effects', () => { }); }); - it('blocks empty saves, updates search state, and quits without saving', () => { + it('prompts skip-empty confirmation on empty save, updates search state, and quits without saving', () => { const state = buildInitialState({ tree: buildPickerTree(pages()), existingRootPageIds: [], currentCrawlMode: 'selected_roots', }); - const blockedSave = reducer(state, 'save-request', 9000); - expect(blockedSave).toEqual({ - next: { - ...state, - transientHint: { - text: 'Select at least one page or press q to quit', - expiresAt: 9000 + TRANSIENT_HINT_DURATION_MS, - }, - }, + const emptySave = reducer(state, 'save-request'); + expect(emptySave).toEqual({ + next: { ...state, pendingConfirm: 'skip-empty' }, + effect: null, + }); + expect(reducer(emptySave.next, 'save-confirm')).toEqual({ + next: { ...state, pendingConfirm: null }, + effect: 'quit-without-save', + }); + expect(reducer(emptySave.next, 'save-cancel')).toEqual({ + next: { ...state, pendingConfirm: null }, effect: null, }); expect( @@ -262,7 +263,7 @@ describe('bulk actions and reducer effects', () => { const withHint = { ...state, transientHint: { - text: 'Select at least one page or press q to quit', + text: 'Select at least one page or press esc to cancel', expiresAt: 11500, }, }; diff --git a/packages/cli/src/notion-page-picker-tree.ts b/packages/cli/src/notion-page-picker-tree.ts index 379ac938..738ab723 100644 --- a/packages/cli/src/notion-page-picker-tree.ts +++ b/packages/cli/src/notion-page-picker-tree.ts @@ -22,7 +22,7 @@ export interface PickerState { checked: Set; cursorId: string; search: { editing: boolean; query: string }; - pendingConfirm: 'mode-switch' | null; + pendingConfirm: 'mode-switch' | 'skip-empty' | null; preLoadWarnings: string[]; transientHint: { text: string; expiresAt: number } | null; currentCrawlMode: 'all_accessible' | 'selected_roots'; @@ -61,7 +61,7 @@ interface MutableNode { childIds: string[]; } -export const TRANSIENT_HINT_DURATION_MS = 2500; +const TRANSIENT_HINT_DURATION_MS = 2500; const collator = new Intl.Collator('en', { sensitivity: 'base', numeric: true }); @@ -444,7 +444,8 @@ export function buildInitialState(args: { export function reducer(state: PickerState, cmd: PickerCommand, now = Date.now()): { next: PickerState; effect: PickerEffect } { if (state.pendingConfirm) { if (cmd === 'save-confirm') { - return { next: cloneState(state, { pendingConfirm: null }), effect: 'save' }; + const effect: PickerEffect = state.pendingConfirm === 'skip-empty' ? 'quit-without-save' : 'save'; + return { next: cloneState(state, { pendingConfirm: null }), effect }; } if (cmd === 'save-cancel') { return { next: cloneState(state, { pendingConfirm: null }), effect: null }; @@ -498,19 +499,13 @@ export function reducer(state: PickerState, cmd: PickerCommand, now = Date.now() }; case 'save-request': if (state.checked.size === 0) { - return { - next: cloneState(state, { - transientHint: transientHint('Select at least one page or press q to quit', now), - }), - effect: null, - }; + return { next: cloneState(state, { pendingConfirm: 'skip-empty' }), effect: null }; } if (state.currentCrawlMode === 'all_accessible') { return { next: cloneState(state, { pendingConfirm: 'mode-switch' }), effect: null }; } return { next: state, effect: 'save' }; case 'save-confirm': - return { next: state, effect: 'save' }; case 'save-cancel': return { next: state, effect: null }; case 'quit': diff --git a/packages/cli/src/notion-page-picker-tui.test.tsx b/packages/cli/src/notion-page-picker-tui.test.tsx index 2d4dffc3..16ad93db 100644 --- a/packages/cli/src/notion-page-picker-tui.test.tsx +++ b/packages/cli/src/notion-page-picker-tui.test.tsx @@ -1,7 +1,7 @@ /* @jsxImportSource react */ import { render as renderInkTest } from 'ink-testing-library'; -import { act, type ReactNode } from 'react'; -import { afterEach, describe, expect, it, vi } from 'vitest'; +import { type ReactNode } from 'react'; +import { describe, expect, it, vi } from 'vitest'; import { buildInitialState, buildPickerTree, type NotionPickerPageInput } from './notion-page-picker-tree.js'; import { NotionPickerApp, @@ -70,13 +70,9 @@ function fakeInkInstance(): NotionPickerInkInstance { } function normalizeFrameWrap(frame: string | undefined): string { - return frame?.replace(/\n/g, ' ') ?? ''; + return frame?.replace(/\n/g, ' ').replace(/│ /g, '').replace(/ +/g, ' ') ?? ''; } -afterEach(() => { - vi.useRealTimers(); -}); - describe('notionPickerCommandForInkInput', () => { it('maps browse, search, and confirm input to reducer commands', () => { expect(notionPickerCommandForInkInput('', { downArrow: true }, state().search, null)).toBe('cursor-down'); @@ -87,9 +83,11 @@ describe('notionPickerCommandForInkInput', () => { expect(notionPickerCommandForInkInput('/', {}, state().search, null)).toBe('search-start'); expect(notionPickerCommandForInkInput('a', {}, state().search, null)).toBe('select-all-visible'); expect(notionPickerCommandForInkInput('n', {}, state().search, null)).toBe('select-none'); - expect(notionPickerCommandForInkInput('s', {}, state().search, null)).toBe('save-request'); - expect(notionPickerCommandForInkInput('q', {}, state().search, null)).toBe('quit'); + expect(notionPickerCommandForInkInput('', { return: true }, state().search, null)).toBe('save-request'); + expect(notionPickerCommandForInkInput('', { escape: true }, state().search, null)).toBe('quit'); expect(notionPickerCommandForInkInput('c', { ctrl: true }, state().search, null)).toBe('quit'); + expect(notionPickerCommandForInkInput('s', {}, state().search, null)).toBeNull(); + expect(notionPickerCommandForInkInput('q', {}, state().search, null)).toBeNull(); expect(notionPickerCommandForInkInput('x', {}, { editing: true, query: '' }, null)).toEqual({ type: 'search-input', @@ -145,13 +143,16 @@ describe('NotionPickerApp', () => { ); const frame = lastFrame() ?? ''; - expect(frame).toContain('Notion pages visible to integration "Design Workspace"'); + expect(frame).toContain('Select Notion pages to ingest'); + expect(frame).toContain('Workspace: Design Workspace'); expect(frame).toContain('5000-page cap reached - some pages not shown'); expect(frame).toContain('1 stored root_page_ids no longer visible - they will be removed if you save'); - expect(frame).toContain('▸ [ ] Engineering Docs ▸ (1)'); - expect(frame).toContain(' [ ] Marketing'); + expect(frame).toContain('◻ Engineering Docs ▸ (1)'); + expect(frame).toContain('◻ Marketing'); expect(frame).not.toContain('Search ready: -'); - expect(frame).toContain('space toggle · enter expand · / search · a all · n none · s save & exit · q quit'); + expect(normalizeFrameWrap(frame)).toContain( + 'Right Arrow to expand, Up/Down to move, Space to select or unselect, Slash to filter, Enter to confirm, Escape to go back, or Ctrl+C to exit.', + ); }); it('renders partial discovery warnings without stale-root save suffix', () => { @@ -199,8 +200,8 @@ describe('NotionPickerApp', () => { ); const frame = lastFrame() ?? ''; - expect(frame).toContain('▸ [×] Engineering Docs ▾'); - expect(frame).toContain(' [~] Architecture'); + expect(frame).toContain('◼ Engineering Docs ▾'); + expect(frame).toContain(' ◼ Architecture'); }); it('supports keyboard selection, all_accessible confirmation, and save callback', async () => { @@ -220,12 +221,12 @@ describe('NotionPickerApp', () => { stdin.write(' '); await waitForInkInput(); - expect(lastFrame()).toContain('[×] Engineering Docs'); + expect(lastFrame()).toContain('◼ Engineering Docs'); - stdin.write('s'); + stdin.write('\r'); await waitForInkInput(); expect(normalizeFrameWrap(lastFrame())).toContain( - 'Save will switch crawl_mode all_accessible -> selected_roots and limit ingest to 1 selected page. [y] confirm [esc] back', + 'Switch crawl_mode from all_accessible to selected_roots? Will limit ingest to 1 selected page. Press Enter to confirm or Escape to go back.', ); stdin.write('y'); @@ -233,8 +234,7 @@ describe('NotionPickerApp', () => { expect(onExit).toHaveBeenCalledWith({ kind: 'save', rootPageIds: [IDS.engineering] }); }); - it('removes transient hints after their expiry time', async () => { - vi.useFakeTimers(); + it('prompts skip-empty confirmation on empty submit and dismisses on cancel', async () => { const onExit = vi.fn(); const { stdin, lastFrame } = renderInkTest( { />, ); - await act(async () => { - stdin.write('s'); - await vi.advanceTimersByTimeAsync(10); - }); - expect(lastFrame()).toContain('Select at least one page or press q to quit'); - - await act(async () => { - await vi.advanceTimersByTimeAsync(2500); - }); - expect(lastFrame()).not.toContain('Select at least one page or press q to quit'); + stdin.write('\r'); + await waitForInkInput(); + expect(normalizeFrameWrap(lastFrame())).toContain( + 'Nothing selected. Skip this step? Press Enter to skip or Escape to go back.', + ); expect(onExit).not.toHaveBeenCalled(); + + stdin.write('n'); + await waitForInkInput(); + expect(lastFrame()).not.toContain('Nothing selected. Skip this step?'); + expect(onExit).not.toHaveBeenCalled(); + + stdin.write('\r'); + await waitForInkInput(); + expect(lastFrame()).toContain('Nothing selected. Skip this step?'); + + stdin.write('\r'); + await waitForInkInput(); + expect(onExit).toHaveBeenCalledWith({ kind: 'quit' }); }); it('renders row-window overflow indicators when the visible list is clipped', async () => { @@ -312,7 +320,7 @@ describe('NotionPickerApp', () => { />, ); - stdin.write('q'); + stdin.write('\u0003'); await waitForInkInput(); expect(onExit).toHaveBeenCalledWith({ kind: 'quit' }); }); diff --git a/packages/cli/src/notion-page-picker-tui.tsx b/packages/cli/src/notion-page-picker-tui.tsx index 30af7522..d627d200 100644 --- a/packages/cli/src/notion-page-picker-tui.tsx +++ b/packages/cli/src/notion-page-picker-tui.tsx @@ -16,6 +16,7 @@ const COLOR_THEME = { text: 'white', muted: 'gray', active: 'cyan', + selected: 'green', warning: 'yellow', } as const; @@ -23,6 +24,7 @@ const NO_COLOR_THEME = { text: 'white', muted: 'white', active: 'white', + selected: 'white', warning: 'white', } as const; @@ -158,13 +160,12 @@ export function notionPickerCommandForInkInput( if (key.downArrow) return 'cursor-down'; if (key.leftArrow) return 'cursor-left'; if (key.rightArrow) return 'cursor-right'; - if (key.return) return 'expand'; + if (key.return) return 'save-request'; if (input === ' ') return 'toggle-check'; if (input === '/') return 'search-start'; if (input === 'a') return 'select-all-visible'; if (input === 'n') return 'select-none'; - if (input === 's') return 'save-request'; - if (input === 'q' || key.escape) return 'quit'; + if (key.escape) return 'quit'; return null; } @@ -174,18 +175,27 @@ function PickerRow(props: { state: PickerState; nodeId: string; width: number; t const focused = props.state.cursorId === node.id; const locked = isAncestorChecked(node.id, props.state.checked, props.state.byId); const checked = props.state.checked.has(node.id); - const glyph = locked ? '[~]' : checked ? '[×]' : '[ ]'; - const children = + const isSelected = checked || locked; + const glyph = isSelected ? '◼' : '◻'; + const glyphColor = locked ? props.theme.muted : checked ? props.theme.selected : props.theme.muted; + const childAffordance = node.childIds.length > 0 ? (props.state.expanded.has(node.id) ? ' ▾' : ` ▸ (${node.childIds.length})`) : ''; - const prefix = `${focused ? '▸' : ' '} ${glyph} ${' '.repeat(node.depth * 2)}`; - const color = focused ? props.theme.active : locked || node.archived ? props.theme.muted : props.theme.text; - const title = truncateText(`${node.title}${children}`, Math.max(10, props.width - prefix.length)); + const indent = ' '.repeat(node.depth * 2); + const titleColor = focused ? props.theme.text : props.theme.muted; const inverse = rowMatchesSearch(props.state, node.id); + const prefixWidth = indent.length + 2; + const title = truncateText(`${node.title}${childAffordance}`, Math.max(10, props.width - prefixWidth)); return ( - - {prefix} - {title} + + + {indent} + {glyph} + + + {' '} + {title} + ); } @@ -198,7 +208,7 @@ export function NotionPickerApp(props: NotionPickerAppProps): ReactNode { const visibleIds = visibleNodeIds(state); const selectedIndex = Math.max(0, visibleIds.indexOf(state.cursorId)); const reservedRows = state.pendingConfirm === 'mode-switch' ? 9 : 8; - const visibleRows = Math.max(5, Math.min(20, (props.terminalRows ?? 24) - reservedRows)); + const visibleRows = Math.max(5, Math.min(12, (props.terminalRows ?? 24) - reservedRows)); const rows = windowItems(visibleIds, selectedIndex, visibleRows); const hiddenAbove = rows.offset; const hiddenBelow = Math.max(0, visibleIds.length - rows.offset - rows.items.length); @@ -254,34 +264,60 @@ export function NotionPickerApp(props: NotionPickerAppProps): ReactNode { return ( - Notion pages visible to integration "{props.workspaceLabel}" - {props.cappedAtCount ? {props.cappedAtCount}-page cap reached - some pages not shown : null} - {state.preLoadWarnings.map((warning) => ( - - {staleWarningText(warning)} - - ))} - {showSearch ? ( + + ◆ + Select Notion pages to ingest + + - / {state.search.query} - {state.search.editing ? '█' : ''} ({searchMatchCount} matches) + Right Arrow to expand, Up/Down to move, Space to select or unselect, Slash to filter, Enter to confirm, Escape + to go back, or Ctrl+C to exit. - ) : null} - + + Workspace: {props.workspaceLabel} + {props.cappedAtCount ? ( + {props.cappedAtCount}-page cap reached - some pages not shown + ) : null} + {state.preLoadWarnings.map((warning) => ( + + {staleWarningText(warning)} + + ))} + {showSearch ? ( + + / + + {state.search.query} + {state.search.editing ? '█' : ''} + + ({searchMatchCount} matches) + + ) : null} {hiddenAbove > 0 ? ↑ {hiddenAbove} more : null} {rows.items.map((nodeId) => ( ))} {hiddenBelow > 0 ? ↓ {hiddenBelow} more : null} + {state.pendingConfirm === 'mode-switch' ? ( + + Switch crawl_mode from all_accessible to selected_roots? Will limit ingest to{' '} + {selectedPageCountText(selectedCount)}. Press Enter to confirm or Escape to go back. + + ) : null} + {state.pendingConfirm === 'skip-empty' ? ( + Nothing selected. Skip this step? Press Enter to skip or Escape to go back. + ) : null} + {state.transientHint ? {state.transientHint.text} : null} - {state.pendingConfirm === 'mode-switch' ? ( - - Save will switch crawl_mode all_accessible -> selected_roots and limit ingest to{' '} - {selectedPageCountText(selectedCount)}. [y] confirm [esc] back - - ) : null} - {state.transientHint ? {state.transientHint.text} : null} - space toggle · enter expand · / search · a all · n none · s save & exit · q quit + └ ); } @@ -323,7 +359,7 @@ export async function renderNotionPickerTui( exitOnCtrlC: false, patchConsole: false, maxFps: 30, - alternateScreen: true, + alternateScreen: false, }, ); await instance.waitUntilExit(); From c2750dd7970be18a18f4cf3fb595258c88be323f Mon Sep 17 00:00:00 2001 From: Luca Martial <48870843+luca-martial@users.noreply.github.com> Date: Wed, 13 May 2026 17:55:25 -0400 Subject: [PATCH 5/7] refactor(cli): hide internal setup options and remove dead flags (#79) Hide advanced/internal `ktx setup` options from --help output using .hideHelp() so the command surface is approachable for new users. Remove the --project, --agent-scope, and --skip-initial-source-ingest flags that are no longer needed. Update docs and tests to match. Co-authored-by: Claude Opus 4.6 (1M context) --- .../content/docs/cli-reference/ktx-setup.mdx | 94 +------- .../docs/getting-started/quickstart.mdx | 2 +- packages/cli/src/commands/setup-commands.ts | 213 ++++++++++-------- packages/cli/src/index.test.ts | 72 +++++- packages/cli/src/setup-agents.ts | 2 +- 5 files changed, 185 insertions(+), 198 deletions(-) diff --git a/docs-site/content/docs/cli-reference/ktx-setup.mdx b/docs-site/content/docs/cli-reference/ktx-setup.mdx index f490988a..59fbe666 100644 --- a/docs-site/content/docs/cli-reference/ktx-setup.mdx +++ b/docs-site/content/docs/cli-reference/ktx-setup.mdx @@ -18,8 +18,6 @@ ktx setup [options] | Flag | Description | Default | |------|-------------|---------| | `--project-dir ` | KTX project directory | `KTX_PROJECT_DIR`, nearest `ktx.yaml`, or cwd | -| `--new` | Create a new KTX project before setup | `false` | -| `--existing` | Use an existing KTX project | `false` | | `--yes` | Accept safe defaults in non-interactive setup | `false` | | `--no-input` | Disable interactive terminal input | — | @@ -29,76 +27,11 @@ ktx setup [options] |------|-------------|---------| | `--agents` | Install agent integration only | `false` | | `--target ` | Agent target (`claude-code`, `codex`, `cursor`, `opencode`, `universal`) | — | -| `--agent-scope ` | Agent install scope (`project` or `global`) | `project` | -| `--project` | Install agent integration into the project scope | `false` | | `--global` | Install agent integration into the global target scope (Claude Code and Codex only) | `false` | -| `--skip-agents` | Leave agent integration incomplete for now | `false` | -### LLM Configuration - -| Flag | Description | Default | -|------|-------------|---------| -| `--anthropic-api-key-env ` | Environment variable containing the Anthropic API key | — | -| `--anthropic-api-key-file ` | File containing the Anthropic API key | — | -| `--anthropic-model ` | Anthropic model ID to validate and save | — | -| `--skip-llm` | Leave LLM setup incomplete for now | `false` | - -### Embedding Configuration - -| Flag | Description | Default | -|------|-------------|---------| -| `--embedding-backend ` | Embedding backend (`openai` or `sentence-transformers`) | — | -| `--embedding-api-key-env ` | Environment variable containing the embedding provider API key | — | -| `--embedding-api-key-file ` | File containing the embedding provider API key | — | -| `--skip-embeddings` | Leave embedding setup incomplete for now | `false` | - -### Database Configuration - -| Flag | Description | Default | -|------|-------------|---------| -| `--database ` | Database driver to configure; repeatable (`sqlite`, `postgres`, `mysql`, `clickhouse`, `sqlserver`, `bigquery`, `snowflake`) | — | -| `--database-connection-id ` | Existing or new connection id; repeatable | — | -| `--new-database-connection-id ` | Connection id for one new database connection | — | -| `--database-url ` | URL, `env:NAME`, or `file:/path` for one new URL-style database connection | — | -| `--database-schema ` | Database schema to include; repeatable | — | -| `--skip-databases` | Leave database setup incomplete | `false` | - -### Historic SQL - -| Flag | Description | Default | -|------|-------------|---------| -| `--enable-historic-sql` | Enable Historic SQL when the selected database supports it | `false` | -| `--disable-historic-sql` | Disable Historic SQL for the selected database | `false` | -| `--historic-sql-window-days ` | Historic SQL query-history window in days | — | -| `--historic-sql-min-executions ` | Minimum executions for a Historic SQL template | — | -| `--historic-sql-min-calls ` | Alias for `--historic-sql-min-executions` for one release | — | -| `--historic-sql-service-account-pattern ` | Historic SQL service-account regex; repeatable | — | -| `--historic-sql-redaction-pattern ` | Historic SQL SQL-literal redaction regex; repeatable | — | - -### Context Source Configuration - -| Flag | Description | Default | -|------|-------------|---------| -| `--source ` | Source connector type (`dbt`, `metricflow`, `metabase`, `looker`, `lookml`, `notion`) | — | -| `--source-connection-id ` | Connection id for source setup | — | -| `--source-path ` | Local source path for dbt, MetricFlow, or LookML | — | -| `--source-git-url ` | Git URL for dbt, MetricFlow, or LookML | — | -| `--source-branch ` | Git branch for source setup | — | -| `--source-subpath ` | Repo subpath for source setup | — | -| `--source-auth-token-ref ` | `env:` or `file:` credential ref for source repo auth | — | -| `--source-url ` | Source service URL for Metabase or Looker | — | -| `--source-api-key-ref ` | `env:` or `file:` API key ref for Metabase or Notion | — | -| `--source-client-id ` | Looker client id | — | -| `--source-client-secret-ref ` | `env:` or `file:` Looker client secret ref | — | -| `--source-warehouse-connection-id ` | Mapped warehouse connection id | — | -| `--source-project-name ` | dbt project name override | — | -| `--source-profiles-path ` | dbt profiles path | — | -| `--source-target ` | dbt target or source-specific mapping target | — | -| `--metabase-database-id ` | Metabase database id to map | — | -| `--notion-crawl-mode ` | Notion crawl mode (`all_accessible` or `selected_roots`) | — | -| `--notion-root-page-id ` | Notion root page id; repeatable | — | -| `--skip-initial-source-ingest` | Validate source setup without building source context during setup | `false` | -| `--skip-sources` | Mark optional source setup complete with no sources | `false` | +The setup wizard is the public configuration interface. It prompts for LLM +credentials, embeddings, database connections, context sources, Historic SQL, +and agent integration when those values are needed. ## Examples @@ -106,17 +39,8 @@ ktx setup [options] # Run the interactive setup wizard ktx setup -# Create a new project and run setup -ktx setup --new - -# Resume setup in an existing project -ktx setup --existing - -# Non-interactive setup with Anthropic key from environment -ktx setup --yes --anthropic-api-key-env ANTHROPIC_API_KEY - -# Set up a Postgres connection -ktx setup --database postgres --database-url "env:DATABASE_URL" +# Run setup for a specific project directory +ktx setup --project-dir ./analytics # Install agent integration for Claude Code only ktx setup --agents --target claude-code @@ -124,12 +48,6 @@ ktx setup --agents --target claude-code # Install agent integration globally for Codex ktx setup --agents --target codex --global -# Add a dbt source from a local path -ktx setup --source dbt --source-path ./my-dbt-project - -# Skip optional steps for a minimal setup -ktx setup --skip-sources --skip-agents - # Check setup readiness ktx status ``` @@ -156,5 +74,5 @@ Agent integration ready: yes (codex:project) |-------|-------|----------| | Setup resumes an unexpected project | `KTX_PROJECT_DIR` or nearest `ktx.yaml` points to another directory | Pass `--project-dir ` explicitly | | Health check for model fails | Provider key or model id is invalid | Set the correct environment variable or secret file and rerun setup | -| Setup cannot run in CI | Interactive prompts need a TTY | Use `--yes --no-input` with explicit flags for required values | +| Setup cannot run in CI | Interactive prompts need a TTY | Run setup interactively before CI, or provide a fixture `ktx.yaml` for automated tests | | Agent integration missing | Setup skipped the agents step | Run `ktx setup --agents --target ` | diff --git a/docs-site/content/docs/getting-started/quickstart.mdx b/docs-site/content/docs/getting-started/quickstart.mdx index 7aba00fd..635c666b 100644 --- a/docs-site/content/docs/getting-started/quickstart.mdx +++ b/docs-site/content/docs/getting-started/quickstart.mdx @@ -242,7 +242,7 @@ Agent integration ready: yes (claude-code:project) | Local embeddings hang or fail | The managed Python runtime cannot start or the local model runtime is unavailable | Install `uv`, run `ktx dev runtime status`, then run `ktx dev runtime install --feature local-embeddings --yes` and rerun setup | | Database connection test fails | Credentials, network access, warehouse, database, or schema value is wrong | Test the same URL with the database's native client, then rerun `ktx setup` and reconfigure the connection | | `KTX context built: no` in `ktx status` | Setup saved configuration but did not build context | Run `ktx setup` and choose to build context now | -| Agent integration is incomplete | Setup skipped the agents step or the target was not installed | Run `ktx setup --agents --target codex --project` using the target you need | +| Agent integration is incomplete | Setup skipped the agents step or the target was not installed | Run `ktx setup --agents --target codex` using the target you need | ## Next steps diff --git a/packages/cli/src/commands/setup-commands.ts b/packages/cli/src/commands/setup-commands.ts index 1688724d..3da8d094 100644 --- a/packages/cli/src/commands/setup-commands.ts +++ b/packages/cli/src/commands/setup-commands.ts @@ -64,13 +64,6 @@ function sourceType(value: string): KtxSetupSourceType { throw new InvalidArgumentError(`invalid choice '${value}'`); } -function agentScope(value: string): 'project' | 'global' { - if (value === 'project' || value === 'global') { - return value; - } - throw new InvalidArgumentError(`invalid choice '${value}'`); -} - function positiveNumber(value: string): number { const parsed = Number.parseInt(value, 10); if (!Number.isInteger(parsed) || parsed <= 0) { @@ -97,7 +90,6 @@ function shouldShowSetupEntryMenu( agents?: boolean; target?: string; global?: boolean; - project?: boolean; skipAgents?: boolean; yes?: boolean; input?: boolean; @@ -142,7 +134,6 @@ function shouldShowSetupEntryMenu( metabaseDatabaseId?: number; notionCrawlMode?: string; notionRootPageId?: string[]; - skipInitialSourceIngest?: boolean; skipSources?: boolean; }, command: Command, @@ -172,7 +163,6 @@ function shouldShowSetupEntryMenu( 'agents', 'target', 'global', - 'project', 'skipAgents', 'yes', 'input', @@ -211,7 +201,6 @@ function shouldShowSetupEntryMenu( 'sourceTarget', 'metabaseDatabaseId', 'notionCrawlMode', - 'skipInitialSourceIngest', 'skipSources', ].some((optionName) => optionWasSpecified(command, optionName)); } @@ -220,9 +209,9 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo const setup = program .command('setup') .description('Set up or resume a local KTX project') - .option('--project-dir ', 'KTX project directory') - .option('--new', 'Create a new KTX project before setup', false) - .option('--existing', 'Use an existing KTX project', false) + .addOption(new Option('--project-dir ', 'KTX project directory').hideHelp()) + .addOption(new Option('--new', 'Create a new KTX project before setup').hideHelp().default(false)) + .addOption(new Option('--existing', 'Use an existing KTX project').hideHelp().default(false)) .option('--agents', 'Install agent integration only', false) .addOption( new Option('--target ', 'Agent target').choices([ @@ -233,94 +222,124 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo 'universal', ]), ) - .addOption(new Option('--agent-scope ', 'Agent install scope').argParser(agentScope).default('project')) - .option('--project', 'Install agent integration into the project scope', false) .option('--global', 'Install agent integration into the global target scope', false) - .option('--skip-agents', 'Leave agent integration incomplete for now', false) + .addOption(new Option('--skip-agents', 'Leave agent integration incomplete for now').hideHelp().default(false)) .option('--yes', 'Accept safe defaults in non-interactive setup', false) .option('--no-input', 'Disable interactive terminal input') - .addOption(new Option('--llm-backend ', 'LLM backend').argParser(llmBackend)) - .option('--anthropic-api-key-env ', 'Environment variable containing the Anthropic API key') - .option('--anthropic-api-key-file ', 'File containing the Anthropic API key') - .option('--anthropic-model ', 'Anthropic model ID to validate and save') - .option('--vertex-project ', 'Google Vertex AI project ID, env:NAME, or file:/path') - .option('--vertex-location ', 'Google Vertex AI location, env:NAME, or file:/path') - .addOption(new Option('--skip-llm', 'Leave LLM setup incomplete for now').hideHelp().default(false)) - .addOption(new Option('--embedding-backend ', 'Embedding backend').argParser(embeddingBackend)) - .option('--embedding-api-key-env ', 'Environment variable containing the embedding provider API key') - .option('--embedding-api-key-file ', 'File containing the embedding provider API key') - .addOption(new Option('--skip-embeddings', 'Leave embedding setup incomplete for now').hideHelp().default(false)) - .option( - '--database ', - 'Database driver to configure; repeatable', - (value, previous: KtxSetupDatabaseDriver[]) => { - return [...previous, databaseDriver(value)]; - }, - [] as KtxSetupDatabaseDriver[], - ) - .option( - '--database-connection-id ', - 'Existing selected connection id or new connection id', - (value, previous: string[]) => [...previous, value], - [], - ) - .option('--new-database-connection-id ', 'Connection id for one new database connection', (value) => { - if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(value)) { - throw new InvalidArgumentError(`Unsafe connection id: ${value}`); - } - return value; - }) - .option('--database-url ', 'URL, env:NAME, or file:/path for one new URL-style database connection') - .option( - '--database-schema ', - 'Database schema to include; repeatable', - (value, previous: string[]) => [...previous, value], - [], - ) - .option('--enable-historic-sql', 'Enable Historic SQL when the selected database supports it', false) - .option('--disable-historic-sql', 'Disable Historic SQL for the selected database', false) - .option('--historic-sql-window-days ', 'Historic SQL query-history window', positiveInteger) - .option('--historic-sql-min-executions ', 'Minimum Historic SQL executions for a template', positiveInteger) - .option( - '--historic-sql-service-account-pattern ', - 'Historic SQL service-account regex; repeatable', - (value, previous: string[]) => [...previous, value], - [], - ) - .option( - '--historic-sql-redaction-pattern ', - 'Historic SQL SQL-literal redaction regex; repeatable', - (value, previous: string[]) => [...previous, value], - [], - ) - .option('--skip-databases', 'Leave database setup incomplete; KTX cannot work until a primary source is added', false) - .addOption(new Option('--source ', 'Source connector type').argParser(sourceType)) - .option('--source-connection-id ', 'Connection id for source setup') - .option('--source-path ', 'Local source path for dbt, MetricFlow, or LookML') - .option('--source-git-url ', 'Git URL for dbt, MetricFlow, or LookML') - .option('--source-branch ', 'Git branch for source setup') - .option('--source-subpath ', 'Repo subpath for source setup') - .option('--source-auth-token-ref ', 'env: or file: credential ref for source repo auth') - .option('--source-url ', 'Source service URL for Metabase or Looker') - .option('--source-api-key-ref ', 'env: or file: API key ref for Metabase or Notion') - .option('--source-client-id ', 'Looker client id') - .option('--source-client-secret-ref ', 'env: or file: Looker client secret ref') - .option('--source-warehouse-connection-id ', 'Mapped warehouse connection id') - .option('--source-project-name ', 'dbt project name override') - .option('--source-profiles-path ', 'dbt profiles path') - .option('--source-target ', 'dbt target or source-specific mapping target') - .option('--metabase-database-id ', 'Metabase database id to map', positiveNumber) + .addOption(new Option('--llm-backend ', 'LLM backend').argParser(llmBackend).hideHelp()) .addOption( - new Option('--notion-crawl-mode ', 'Notion crawl mode').choices(['all_accessible', 'selected_roots']), + new Option('--anthropic-api-key-env ', 'Environment variable containing the Anthropic API key').hideHelp(), ) - .option( - '--notion-root-page-id ', - 'Notion root page id; repeatable', - (value, previous: string[]) => [...previous, value], - [], + .addOption( + new Option('--anthropic-api-key-file ', 'File containing the Anthropic API key').hideHelp(), ) - .option('--skip-initial-source-ingest', 'Validate source setup without building source context during setup', false) - .option('--skip-sources', 'Mark optional source setup complete with no sources', false) + .addOption(new Option('--anthropic-model ', 'Anthropic model ID to validate and save').hideHelp()) + .addOption(new Option('--vertex-project ', 'Google Vertex AI project ID, env:NAME, or file:/path').hideHelp()) + .addOption(new Option('--vertex-location ', 'Google Vertex AI location, env:NAME, or file:/path').hideHelp()) + .addOption(new Option('--skip-llm', 'Leave LLM setup incomplete for now').hideHelp().default(false)) + .addOption(new Option('--embedding-backend ', 'Embedding backend').argParser(embeddingBackend).hideHelp()) + .addOption( + new Option( + '--embedding-api-key-env ', + 'Environment variable containing the embedding provider API key', + ).hideHelp(), + ) + .addOption( + new Option('--embedding-api-key-file ', 'File containing the embedding provider API key').hideHelp(), + ) + .addOption(new Option('--skip-embeddings', 'Leave embedding setup incomplete for now').hideHelp().default(false)) + .addOption( + new Option('--database ', 'Database driver to configure; repeatable') + .argParser((value, previous: KtxSetupDatabaseDriver[]) => { + return [...previous, databaseDriver(value)]; + }) + .default([] as KtxSetupDatabaseDriver[]) + .hideHelp(), + ) + .addOption( + new Option('--database-connection-id ', 'Existing selected connection id or new connection id') + .argParser((value, previous: string[]) => [...previous, value]) + .default([] as string[]) + .hideHelp(), + ) + .addOption( + new Option('--new-database-connection-id ', 'Connection id for one new database connection') + .argParser((value) => { + if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(value)) { + throw new InvalidArgumentError(`Unsafe connection id: ${value}`); + } + return value; + }) + .hideHelp(), + ) + .addOption( + new Option('--database-url ', 'URL, env:NAME, or file:/path for one new URL-style database connection').hideHelp(), + ) + .addOption( + new Option('--database-schema ', 'Database schema to include; repeatable') + .argParser((value, previous: string[]) => [...previous, value]) + .default([] as string[]) + .hideHelp(), + ) + .addOption( + new Option('--enable-historic-sql', 'Enable Historic SQL when the selected database supports it') + .hideHelp() + .default(false), + ) + .addOption( + new Option('--disable-historic-sql', 'Disable Historic SQL for the selected database').hideHelp().default(false), + ) + .addOption(new Option('--historic-sql-window-days ', 'Historic SQL query-history window').argParser(positiveInteger).hideHelp()) + .addOption( + new Option('--historic-sql-min-executions ', 'Minimum Historic SQL executions for a template') + .argParser(positiveInteger) + .hideHelp(), + ) + .addOption( + new Option('--historic-sql-service-account-pattern ', 'Historic SQL service-account regex; repeatable') + .argParser((value, previous: string[]) => [...previous, value]) + .default([] as string[]) + .hideHelp(), + ) + .addOption( + new Option('--historic-sql-redaction-pattern ', 'Historic SQL SQL-literal redaction regex; repeatable') + .argParser((value, previous: string[]) => [...previous, value]) + .default([] as string[]) + .hideHelp(), + ) + .addOption( + new Option('--skip-databases', 'Leave database setup incomplete; KTX cannot work until a primary source is added') + .hideHelp() + .default(false), + ) + .addOption(new Option('--source ', 'Source connector type').argParser(sourceType).hideHelp()) + .addOption(new Option('--source-connection-id ', 'Connection id for source setup').hideHelp()) + .addOption(new Option('--source-path ', 'Local source path for dbt, MetricFlow, or LookML').hideHelp()) + .addOption(new Option('--source-git-url ', 'Git URL for dbt, MetricFlow, or LookML').hideHelp()) + .addOption(new Option('--source-branch ', 'Git branch for source setup').hideHelp()) + .addOption(new Option('--source-subpath ', 'Repo subpath for source setup').hideHelp()) + .addOption(new Option('--source-auth-token-ref ', 'env: or file: credential ref for source repo auth').hideHelp()) + .addOption(new Option('--source-url ', 'Source service URL for Metabase or Looker').hideHelp()) + .addOption(new Option('--source-api-key-ref ', 'env: or file: API key ref for Metabase or Notion').hideHelp()) + .addOption(new Option('--source-client-id ', 'Looker client id').hideHelp()) + .addOption(new Option('--source-client-secret-ref ', 'env: or file: Looker client secret ref').hideHelp()) + .addOption(new Option('--source-warehouse-connection-id ', 'Mapped warehouse connection id').hideHelp()) + .addOption(new Option('--source-project-name ', 'dbt project name override').hideHelp()) + .addOption(new Option('--source-profiles-path ', 'dbt profiles path').hideHelp()) + .addOption(new Option('--source-target ', 'dbt target or source-specific mapping target').hideHelp()) + .addOption(new Option('--metabase-database-id ', 'Metabase database id to map').argParser(positiveNumber).hideHelp()) + .addOption( + new Option('--notion-crawl-mode ', 'Notion crawl mode') + .choices(['all_accessible', 'selected_roots']) + .hideHelp(), + ) + .addOption( + new Option('--notion-root-page-id ', 'Notion root page id; repeatable') + .argParser((value, previous: string[]) => [...previous, value]) + .default([] as string[]) + .hideHelp(), + ) + .addOption(new Option('--skip-sources', 'Mark optional source setup complete with no sources').hideHelp().default(false)) .showHelpAfterError(); setup.hook('preAction', (_thisCommand, actionCommand) => { @@ -371,7 +390,7 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo } const mode = options.new ? 'new' : options.existing ? 'existing' : 'auto'; - const resolvedAgentScope = options.global ? 'global' : options.agentScope; + const resolvedAgentScope = options.global ? 'global' : 'project'; await runSetupArgs(context, { command: 'run', projectDir: resolveCommandProjectDir(command), diff --git a/packages/cli/src/index.test.ts b/packages/cli/src/index.test.ts index cd635d78..305cf30e 100644 --- a/packages/cli/src/index.test.ts +++ b/packages/cli/src/index.test.ts @@ -444,20 +444,54 @@ describe('runKtxCli', () => { expect(io.stderr()).toContain('Choose only one runtime install mode: --yes or --no-input'); }); - it('documents setup as a bare command without subcommands', async () => { + it('documents setup with only the common interactive options visible', async () => { const testIo = makeIo(); await expect(runKtxCli(['setup', '--help'], testIo.io)).resolves.toBe(0); - expect(testIo.stdout()).toContain('Usage: ktx setup [options]'); - expect(testIo.stdout()).not.toContain('Commands:'); - expect(testIo.stdout()).not.toContain('setup demo'); - expect(testIo.stdout()).not.toContain('setup context'); - expect(testIo.stdout()).not.toContain('--skip-llm'); - expect(testIo.stdout()).not.toContain('--skip-embeddings'); - expect(testIo.stdout()).not.toContain('--embedding-model'); - expect(testIo.stdout()).not.toContain('--embedding-dimensions'); - expect(testIo.stdout()).not.toContain('--embedding-base-url'); + const stdout = testIo.stdout(); + expect(stdout).toContain('Usage: ktx setup [options]'); + expect(stdout).toContain('--agents'); + expect(stdout).toContain('--target '); + expect(stdout).toContain('--global'); + expect(stdout).toContain('--yes'); + expect(stdout).toContain('--no-input'); + expect(stdout).toContain('Global Options:'); + expect(stdout.match(/--project-dir /g)).toHaveLength(1); + expect(stdout).not.toContain('Commands:'); + expect(stdout).not.toContain('setup demo'); + expect(stdout).not.toContain('setup context'); + + for (const hiddenFlag of [ + '--new', + '--existing', + '--agent-scope', + '--skip-agents', + '--llm-backend', + '--anthropic-api-key-env', + '--vertex-project', + '--embedding-backend', + '--database ', + '--database-connection-id', + '--new-database-connection-id', + '--enable-historic-sql', + '--historic-sql-min-executions', + '--skip-databases', + '--source ', + '--source-connection-id', + '--metabase-database-id', + '--notion-root-page-id', + '--skip-initial-source-ingest', + '--skip-sources', + '--skip-llm', + '--skip-embeddings', + '--embedding-model', + '--embedding-dimensions', + '--embedding-base-url', + ]) { + expect(stdout).not.toContain(hiddenFlag); + } + expect(stdout).not.toMatch(/^ --project\s/m); expect(testIo.stderr()).toBe(''); }); @@ -725,6 +759,23 @@ describe('runKtxCli', () => { expect(setup).not.toHaveBeenCalled(); }); + it('rejects removed setup options', async () => { + const setup = vi.fn(async () => 0); + const cases = [ + ['setup', '--project'], + ['setup', '--agent-scope', 'global'], + ['setup', '--skip-initial-source-ingest'], + ]; + + for (const args of cases) { + const testIo = makeIo(); + await expect(runKtxCli(['--project-dir', tempDir, ...args], testIo.io, { setup })).resolves.toBe(1); + expect(testIo.stderr()).toMatch(/unknown option|error:/i); + } + + expect(setup).not.toHaveBeenCalled(); + }); + it('prints ingest help without invoking ingest execution', async () => { const testIo = makeIo(); const ingest = vi.fn(); @@ -1250,7 +1301,6 @@ describe('runKtxCli', () => { '--agents', '--target', 'codex', - '--project', '--no-input', '--yes', ], diff --git a/packages/cli/src/setup-agents.ts b/packages/cli/src/setup-agents.ts index 9505307d..7a18a969 100644 --- a/packages/cli/src/setup-agents.ts +++ b/packages/cli/src/setup-agents.ts @@ -82,7 +82,7 @@ export function plannedKtxAgentFiles(input: { { kind: 'file', path: join(codexHome, 'instructions/ktx.md'), role: 'rule' as const }, ]; } - throw new Error(`Global ${input.target} installation is not supported; use --project.`); + throw new Error(`Global ${input.target} installation is not supported; omit --global.`); } const root = resolve(input.projectDir); From dabd640cade793288812e6fe90b0ab68a3093ae9 Mon Sep 17 00:00:00 2001 From: Luca Martial <48870843+luca-martial@users.noreply.github.com> Date: Wed, 13 May 2026 18:41:44 -0400 Subject: [PATCH 6/7] feat(cli): tree-picker UI for database scope selection (#81) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor(cli): extract generic tree picker from Notion-specific modules Rename notion-page-picker-tree → tree-picker-state and notion-page-picker-tui → tree-picker-tui, removing Notion-specific naming so the tree picker can be reused for database scope selection. Update notion-page-picker to consume the new generic interfaces. Co-Authored-By: Claude Opus 4.6 (1M context) * feat(cli): add database tree picker for schema and table scope selection Replace inline multiselect prompts in setup-databases with a new database-tree-picker that uses the generic tree picker TUI. This gives database scope selection the same grouped tree UI as the Notion page picker, combining schema and table selection into a single step. Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Claude Opus 4.6 (1M context) --- packages/cli/src/database-tree-picker.test.ts | 188 +++++++++ packages/cli/src/database-tree-picker.ts | 210 ++++++++++ .../cli/src/notion-page-picker-tui.test.tsx | 392 ------------------ packages/cli/src/notion-page-picker.test.ts | 50 ++- packages/cli/src/notion-page-picker.ts | 103 +++-- packages/cli/src/setup-databases.test.ts | 160 +++++-- packages/cli/src/setup-databases.ts | 332 ++++++--------- ...tree.test.ts => tree-picker-state.test.ts} | 89 ++-- ...ge-picker-tree.ts => tree-picker-state.ts} | 91 ++-- packages/cli/src/tree-picker-tui.test.tsx | 361 ++++++++++++++++ ...age-picker-tui.tsx => tree-picker-tui.tsx} | 157 +++---- 11 files changed, 1299 insertions(+), 834 deletions(-) create mode 100644 packages/cli/src/database-tree-picker.test.ts create mode 100644 packages/cli/src/database-tree-picker.ts delete mode 100644 packages/cli/src/notion-page-picker-tui.test.tsx rename packages/cli/src/{notion-page-picker-tree.test.ts => tree-picker-state.test.ts} (78%) rename packages/cli/src/{notion-page-picker-tree.ts => tree-picker-state.ts} (86%) create mode 100644 packages/cli/src/tree-picker-tui.test.tsx rename packages/cli/src/{notion-page-picker-tui.tsx => tree-picker-tui.tsx} (67%) diff --git a/packages/cli/src/database-tree-picker.test.ts b/packages/cli/src/database-tree-picker.test.ts new file mode 100644 index 00000000..5559ee42 --- /dev/null +++ b/packages/cli/src/database-tree-picker.test.ts @@ -0,0 +1,188 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + pickDatabaseScope, + type DatabaseTreePickerRenderer, + type PickDatabaseScopeArgs, +} from './database-tree-picker.js'; +import type { TreePickerChrome, TreePickerResult } from './tree-picker-tui.js'; +import type { PickerState } from './tree-picker-state.js'; + +function makeIo() { + let stdout = ''; + let stderr = ''; + return { + io: { + stdout: { isTTY: true, write: (chunk: string) => { stdout += chunk; } }, + stderr: { write: (chunk: string) => { stderr += chunk; } }, + }, + stdout: () => stdout, + stderr: () => stderr, + }; +} + +function captureRenderer(): { + renderer: DatabaseTreePickerRenderer; + capture: { chrome?: TreePickerChrome; state?: PickerState }; + setResult: (result: TreePickerResult) => void; +} { + const capture: { chrome?: TreePickerChrome; state?: PickerState } = {}; + let nextResult: TreePickerResult = { kind: 'quit' }; + const renderer: DatabaseTreePickerRenderer = vi.fn(async (chrome, state) => { + capture.chrome = chrome; + capture.state = state; + return nextResult; + }); + return { + renderer, + capture, + setResult: (result) => { + nextResult = result; + }, + }; +} + +const discovered = [ + { schema: 'analytics', name: 'customers', kind: 'table' as const }, + { schema: 'analytics', name: 'orders', kind: 'table' as const }, + { schema: 'public', name: 'events', kind: 'view' as const }, + { schema: 'public', name: 'sessions', kind: 'table' as const }, +]; + +function baseArgs(overrides: Partial = {}): PickDatabaseScopeArgs { + return { + connectionId: 'warehouse', + schemaNoun: 'schema', + schemaNounPlural: 'schemas', + discovered, + existing: { enabledTables: [] }, + defaultSchemas: ['analytics'], + supportsSchemaScope: true, + ...overrides, + }; +} + +describe('pickDatabaseScope', () => { + it('builds a 2-level tree (schemas as parents, tables as children) and uses save-empty action', async () => { + const { renderer, capture, setResult } = captureRenderer(); + setResult({ kind: 'quit' }); + + await pickDatabaseScope(baseArgs(), makeIo().io, renderer); + + expect(capture.state?.skipEmptyAction).toBe('save-empty'); + const schemaIds = capture.state?.tree.filter((n) => n.parentId === null).map((n) => n.id); + const tableIds = capture.state?.tree.filter((n) => n.parentId !== null).map((n) => n.id); + expect((schemaIds ?? []).sort()).toEqual(['analytics', 'public']); + expect((tableIds ?? []).sort()).toEqual([ + 'analytics.customers', + 'analytics.orders', + 'public.events', + 'public.sessions', + ]); + expect(capture.state?.byId.get('public.events')?.title).toBe('events (view)'); + }); + + it('pre-checks default schemas at the parent level when no existing selection', async () => { + const { renderer, capture, setResult } = captureRenderer(); + setResult({ kind: 'quit' }); + + await pickDatabaseScope(baseArgs({ defaultSchemas: ['analytics'] }), makeIo().io, renderer); + + expect([...(capture.state?.checked ?? [])]).toEqual(['analytics']); + }); + + it('collapses an existing full-schema selection back into the parent check', async () => { + const { renderer, capture, setResult } = captureRenderer(); + setResult({ kind: 'quit' }); + + await pickDatabaseScope( + baseArgs({ existing: { enabledTables: ['analytics.customers', 'analytics.orders'] } }), + makeIo().io, + renderer, + ); + + expect([...(capture.state?.checked ?? [])]).toEqual(['analytics']); + }); + + it('keeps a partial existing selection at the leaf level', async () => { + const { renderer, capture, setResult } = captureRenderer(); + setResult({ kind: 'quit' }); + + await pickDatabaseScope( + baseArgs({ existing: { enabledTables: ['analytics.customers'] } }), + makeIo().io, + renderer, + ); + + expect([...(capture.state?.checked ?? [])]).toEqual(['analytics.customers']); + }); + + it('expands a selected schema parent into all its tables and derives activeSchemas', async () => { + const { renderer, setResult } = captureRenderer(); + setResult({ kind: 'save', selectedIds: ['analytics'] }); + + const result = await pickDatabaseScope(baseArgs(), makeIo().io, renderer); + + expect(result).toEqual({ + kind: 'selected', + activeSchemas: ['analytics'], + enabledTables: ['analytics.customers', 'analytics.orders'], + }); + }); + + it('combines parent and individual leaf selections without duplicate tables', async () => { + const { renderer, setResult } = captureRenderer(); + setResult({ kind: 'save', selectedIds: ['analytics', 'public.events'] }); + + const result = await pickDatabaseScope(baseArgs(), makeIo().io, renderer); + + expect(result).toEqual({ + kind: 'selected', + activeSchemas: ['analytics', 'public'], + enabledTables: ['analytics.customers', 'analytics.orders', 'public.events'], + }); + }); + + it('treats empty save as enable-all', async () => { + const { renderer, setResult } = captureRenderer(); + setResult({ kind: 'save', selectedIds: [] }); + + const result = await pickDatabaseScope(baseArgs(), makeIo().io, renderer); + + expect(result).toEqual({ + kind: 'selected', + activeSchemas: ['analytics', 'public'], + enabledTables: [ + 'analytics.customers', + 'analytics.orders', + 'public.events', + 'public.sessions', + ], + }); + }); + + it('omits activeSchemas when the driver does not support a schema scope', async () => { + const { renderer, setResult } = captureRenderer(); + setResult({ kind: 'save', selectedIds: ['analytics'] }); + + const result = await pickDatabaseScope( + baseArgs({ supportsSchemaScope: false }), + makeIo().io, + renderer, + ); + + expect(result).toEqual({ + kind: 'selected', + activeSchemas: [], + enabledTables: ['analytics.customers', 'analytics.orders'], + }); + }); + + it('returns back when the picker quits', async () => { + const { renderer, setResult } = captureRenderer(); + setResult({ kind: 'quit' }); + + const result = await pickDatabaseScope(baseArgs(), makeIo().io, renderer); + + expect(result).toEqual({ kind: 'back' }); + }); +}); diff --git a/packages/cli/src/database-tree-picker.ts b/packages/cli/src/database-tree-picker.ts new file mode 100644 index 00000000..d494003d --- /dev/null +++ b/packages/cli/src/database-tree-picker.ts @@ -0,0 +1,210 @@ +import type { KtxTableListEntry } from '@ktx/context/scan'; +import type { KtxCliIo } from './cli-runtime.js'; +import { profileMark } from './startup-profile.js'; +import { + buildInitialState, + buildPickerTree, + type PickerState, + type TreePickerNode, + type TreePickerNodeInput, +} from './tree-picker-state.js'; +import { + renderTreePickerTui, + type TreePickerChrome, + type TreePickerResult, + type TreePickerTuiIo, +} from './tree-picker-tui.js'; + +profileMark('module:database-tree-picker'); + +const DATABASE_SCRIPTED_MODE_HINT = + 'Database picker requires a TTY. Use --no-input and the relevant flags for scripted mode.'; + +export type DatabaseTreePickerRenderer = ( + chrome: TreePickerChrome, + initialState: PickerState, + io: TreePickerTuiIo, +) => Promise; + +function defaultRenderer( + chrome: TreePickerChrome, + initialState: PickerState, + io: TreePickerTuiIo, +): Promise { + return renderTreePickerTui({ chrome, initialState }, io, { scriptedModeHint: DATABASE_SCRIPTED_MODE_HINT }); +} + +export type DatabaseScopePickResult = + | { kind: 'selected'; activeSchemas: string[]; enabledTables: string[] } + | { kind: 'back' }; + +export interface PickDatabaseScopeArgs { + connectionId: string; + schemaNoun: string; + schemaNounPlural: string; + discovered: readonly KtxTableListEntry[]; + existing: { enabledTables: readonly string[] }; + defaultSchemas: readonly string[]; + supportsSchemaScope: boolean; +} + +function qualifiedTableId(entry: KtxTableListEntry): string { + return `${entry.schema}.${entry.name}`; +} + +function tableTitle(entry: KtxTableListEntry): string { + return entry.kind === 'view' ? `${entry.name} (view)` : entry.name; +} + +function buildTreeInputs(discovered: readonly KtxTableListEntry[]): { + inputs: TreePickerNodeInput[]; + schemaIds: string[]; + allTables: string[]; +} { + const schemaSeen = new Set(); + const schemaIds: string[] = []; + for (const entry of discovered) { + if (!schemaSeen.has(entry.schema)) { + schemaSeen.add(entry.schema); + schemaIds.push(entry.schema); + } + } + const inputs: TreePickerNodeInput[] = []; + for (const schema of schemaIds) { + inputs.push({ id: schema, title: schema, archived: false, parentId: null }); + } + for (const entry of discovered) { + inputs.push({ + id: qualifiedTableId(entry), + title: tableTitle(entry), + archived: false, + parentId: entry.schema, + }); + } + return { inputs, schemaIds, allTables: discovered.map(qualifiedTableId) }; +} + +function initialSelectionForExisting( + existing: readonly string[], + byId: Map, +): string[] { + const tableIds = new Set( + [...byId.values()].filter((node) => node.parentId !== null).map((node) => node.id), + ); + const existingTables = new Set(existing.filter((id) => tableIds.has(id))); + const schemaChildren = new Map(); + for (const node of byId.values()) { + if (node.parentId === null && node.childIds.length > 0) { + schemaChildren.set(node.id, [...node.childIds]); + } + } + const result: string[] = []; + for (const [schema, children] of schemaChildren) { + const allChecked = children.length > 0 && children.every((childId) => existingTables.has(childId)); + if (allChecked) { + result.push(schema); + for (const childId of children) { + existingTables.delete(childId); + } + } + } + for (const id of existingTables) { + result.push(id); + } + return result; +} + +function initialSelectionFromDefaults( + defaultSchemas: readonly string[], + schemaIds: readonly string[], +): string[] { + const valid = new Set(schemaIds); + const filtered = defaultSchemas.filter((s) => valid.has(s)); + return filtered.length > 0 ? filtered : [...schemaIds]; +} + +function expandSelectedToTables( + selectedIds: readonly string[], + byId: Map, +): string[] { + const expanded: string[] = []; + const seen = new Set(); + for (const id of selectedIds) { + const node = byId.get(id); + if (!node) continue; + if (node.childIds.length === 0) { + if (node.parentId !== null && !seen.has(id)) { + seen.add(id); + expanded.push(id); + } + continue; + } + for (const childId of node.childIds) { + if (!seen.has(childId)) { + seen.add(childId); + expanded.push(childId); + } + } + } + return expanded; +} + +function schemasFromEnabledTables(enabledTables: readonly string[]): string[] { + const seen = new Set(); + const result: string[] = []; + for (const qualified of enabledTables) { + const schema = qualified.split('.')[0] ?? ''; + if (schema.length === 0 || seen.has(schema)) continue; + seen.add(schema); + result.push(schema); + } + return result; +} + +export async function pickDatabaseScope( + args: PickDatabaseScopeArgs, + io: KtxCliIo, + render: DatabaseTreePickerRenderer = defaultRenderer, +): Promise { + const { inputs, schemaIds, allTables } = buildTreeInputs(args.discovered); + const tree = buildPickerTree(inputs); + const byId = new Map(tree.map((node) => [node.id, node])); + const tableCount = allTables.length; + const schemaCount = schemaIds.length; + + const initialSelection = + args.existing.enabledTables.length > 0 + ? initialSelectionForExisting(args.existing.enabledTables, byId) + : initialSelectionFromDefaults(args.defaultSchemas, schemaIds); + + const initialState = buildInitialState({ + tree, + existingSelectedIds: initialSelection, + skipEmptyAction: 'save-empty', + }); + + const schemaWordPlural = schemaCount === 1 ? args.schemaNoun : args.schemaNounPlural; + const subtitleLines = [ + `Connection: ${args.connectionId}`, + `Found ${tableCount} ${tableCount === 1 ? 'table' : 'tables'} across ${schemaCount} ${schemaWordPlural}.`, + `Toggle a ${args.schemaNoun} to enable all of its tables, or expand to pick individual tables.`, + ]; + + const chrome: TreePickerChrome = { + title: `Choose tables to enable for ${args.connectionId}`, + subtitleLines, + skipEmptyMessage: + 'Nothing selected. Enable all tables? Press Enter to enable all or Escape to go back.', + }; + + const result = await render(chrome, initialState, io as TreePickerTuiIo); + if (result.kind === 'quit') { + return { kind: 'back' }; + } + + const enabledTables = + result.selectedIds.length === 0 ? allTables : expandSelectedToTables(result.selectedIds, byId); + const activeSchemas = args.supportsSchemaScope ? schemasFromEnabledTables(enabledTables) : []; + + return { kind: 'selected', activeSchemas, enabledTables }; +} diff --git a/packages/cli/src/notion-page-picker-tui.test.tsx b/packages/cli/src/notion-page-picker-tui.test.tsx deleted file mode 100644 index 16ad93db..00000000 --- a/packages/cli/src/notion-page-picker-tui.test.tsx +++ /dev/null @@ -1,392 +0,0 @@ -/* @jsxImportSource react */ -import { render as renderInkTest } from 'ink-testing-library'; -import { type ReactNode } from 'react'; -import { describe, expect, it, vi } from 'vitest'; -import { buildInitialState, buildPickerTree, type NotionPickerPageInput } from './notion-page-picker-tree.js'; -import { - NotionPickerApp, - notionPickerCommandForInkInput, - renderNotionPickerTui, - resolveNotionPickerWidth, - sanitizeNotionPickerTuiError, - windowItems, - windowOffset, - type NotionPickerInkInstance, - type NotionPickerInkRenderOptions, -} from './notion-page-picker-tui.js'; - -const IDS = { - engineering: '11111111-1111-1111-1111-111111111111', - architecture: '22222222-2222-2222-2222-222222222222', - marketing: '33333333-3333-3333-3333-333333333333', - finance: '44444444-4444-4444-4444-444444444444', - ops: '55555555-5555-5555-5555-555555555555', - sales: '66666666-6666-6666-6666-666666666666', - support: '77777777-7777-7777-7777-777777777777', - product: '88888888-8888-8888-8888-888888888888', - design: '99999999-9999-9999-9999-999999999999', -}; - -function pages(): NotionPickerPageInput[] { - return [ - { id: IDS.engineering, title: 'Engineering Docs', archived: false, parentId: null }, - { id: IDS.architecture, title: 'Architecture', archived: false, parentId: IDS.engineering }, - { id: IDS.marketing, title: 'Marketing', archived: false, parentId: null }, - ]; -} - -function manyPages(): NotionPickerPageInput[] { - return [ - { id: IDS.engineering, title: 'Engineering Docs', archived: false, parentId: null }, - { id: IDS.architecture, title: 'Architecture', archived: false, parentId: IDS.engineering }, - { id: IDS.marketing, title: 'Marketing', archived: false, parentId: null }, - { id: IDS.finance, title: 'Finance', archived: false, parentId: null }, - { id: IDS.ops, title: 'Operations', archived: false, parentId: null }, - { id: IDS.sales, title: 'Sales', archived: false, parentId: null }, - { id: IDS.support, title: 'Support', archived: false, parentId: null }, - { id: IDS.product, title: 'Product', archived: false, parentId: null }, - { id: IDS.design, title: 'Design', archived: false, parentId: null }, - ]; -} - -function state(mode: 'all_accessible' | 'selected_roots' = 'selected_roots') { - return buildInitialState({ - tree: buildPickerTree(pages()), - existingRootPageIds: [], - currentCrawlMode: mode, - }); -} - -async function waitForInkInput(): Promise { - await new Promise((resolve) => setTimeout(resolve, 10)); -} - -function fakeInkInstance(): NotionPickerInkInstance { - return { - rerender: vi.fn(), - unmount: vi.fn(), - waitUntilExit: vi.fn(async () => undefined), - }; -} - -function normalizeFrameWrap(frame: string | undefined): string { - return frame?.replace(/\n/g, ' ').replace(/│ /g, '').replace(/ +/g, ' ') ?? ''; -} - -describe('notionPickerCommandForInkInput', () => { - it('maps browse, search, and confirm input to reducer commands', () => { - expect(notionPickerCommandForInkInput('', { downArrow: true }, state().search, null)).toBe('cursor-down'); - expect(notionPickerCommandForInkInput('', { upArrow: true }, state().search, null)).toBe('cursor-up'); - expect(notionPickerCommandForInkInput('', { rightArrow: true }, state().search, null)).toBe('cursor-right'); - expect(notionPickerCommandForInkInput('', { leftArrow: true }, state().search, null)).toBe('cursor-left'); - expect(notionPickerCommandForInkInput(' ', {}, state().search, null)).toBe('toggle-check'); - expect(notionPickerCommandForInkInput('/', {}, state().search, null)).toBe('search-start'); - expect(notionPickerCommandForInkInput('a', {}, state().search, null)).toBe('select-all-visible'); - expect(notionPickerCommandForInkInput('n', {}, state().search, null)).toBe('select-none'); - expect(notionPickerCommandForInkInput('', { return: true }, state().search, null)).toBe('save-request'); - expect(notionPickerCommandForInkInput('', { escape: true }, state().search, null)).toBe('quit'); - expect(notionPickerCommandForInkInput('c', { ctrl: true }, state().search, null)).toBe('quit'); - expect(notionPickerCommandForInkInput('s', {}, state().search, null)).toBeNull(); - expect(notionPickerCommandForInkInput('q', {}, state().search, null)).toBeNull(); - - expect(notionPickerCommandForInkInput('x', {}, { editing: true, query: '' }, null)).toEqual({ - type: 'search-input', - value: 'x', - }); - expect(notionPickerCommandForInkInput('', { backspace: true }, { editing: true, query: 'x' }, null)).toBe( - 'search-backspace', - ); - expect(notionPickerCommandForInkInput('', { return: true }, { editing: true, query: 'x' }, null)).toBe( - 'search-submit', - ); - expect(notionPickerCommandForInkInput('', { escape: true }, { editing: true, query: 'x' }, null)).toBe( - 'search-cancel', - ); - - expect(notionPickerCommandForInkInput('y', {}, state().search, 'mode-switch')).toBe('save-confirm'); - expect(notionPickerCommandForInkInput('', { return: true }, state().search, 'mode-switch')).toBe('save-confirm'); - expect(notionPickerCommandForInkInput('n', {}, state().search, 'mode-switch')).toBe('save-cancel'); - }); -}); - -describe('window helpers', () => { - it('centers the selected row and returns the visible slice', () => { - expect(windowOffset(20, 10, 5)).toBe(8); - expect(windowItems(['a', 'b', 'c', 'd', 'e'], 3, 3)).toEqual({ items: ['c', 'd', 'e'], offset: 2 }); - }); - - it('clamps picker width to the design rule', () => { - expect(resolveNotionPickerWidth(200)).toBe(120); - expect(resolveNotionPickerWidth(100)).toBe(96); - expect(resolveNotionPickerWidth(50)).toBe(60); - expect(resolveNotionPickerWidth(undefined)).toBe(96); - }); -}); - -describe('NotionPickerApp', () => { - it('renders spec banners, row glyphs, search visibility, and hint text', () => { - const initialState = { - ...state('all_accessible'), - preLoadWarnings: ['1 stored root_page_ids no longer visible'], - }; - const { lastFrame } = renderInkTest( - , - ); - - const frame = lastFrame() ?? ''; - expect(frame).toContain('Select Notion pages to ingest'); - expect(frame).toContain('Workspace: Design Workspace'); - expect(frame).toContain('5000-page cap reached - some pages not shown'); - expect(frame).toContain('1 stored root_page_ids no longer visible - they will be removed if you save'); - expect(frame).toContain('◻ Engineering Docs ▸ (1)'); - expect(frame).toContain('◻ Marketing'); - expect(frame).not.toContain('Search ready: -'); - expect(normalizeFrameWrap(frame)).toContain( - 'Right Arrow to expand, Up/Down to move, Space to select or unselect, Slash to filter, Enter to confirm, Escape to go back, or Ctrl+C to exit.', - ); - }); - - it('renders partial discovery warnings without stale-root save suffix', () => { - const initialState = { - ...state(), - preLoadWarnings: ['Notion search stopped early: rate limit after first page'], - }; - const { lastFrame } = renderInkTest( - , - ); - - const frame = lastFrame() ?? ''; - expect(frame).toContain('Notion search stopped early: rate limit after first page'); - expect(frame).not.toContain( - 'Notion search stopped early: rate limit after first page - they will be removed if you save', - ); - }); - - it('renders checked parents and locked descendants with the locked design glyphs', () => { - const initialState = { - ...state(), - checked: new Set([IDS.engineering]), - expanded: new Set([IDS.engineering]), - }; - const { lastFrame } = renderInkTest( - , - ); - - const frame = lastFrame() ?? ''; - expect(frame).toContain('◼ Engineering Docs ▾'); - expect(frame).toContain(' ◼ Architecture'); - }); - - it('supports keyboard selection, all_accessible confirmation, and save callback', async () => { - const onExit = vi.fn(); - const { stdin, lastFrame } = renderInkTest( - , - ); - - stdin.write(' '); - await waitForInkInput(); - expect(lastFrame()).toContain('◼ Engineering Docs'); - - stdin.write('\r'); - await waitForInkInput(); - expect(normalizeFrameWrap(lastFrame())).toContain( - 'Switch crawl_mode from all_accessible to selected_roots? Will limit ingest to 1 selected page. Press Enter to confirm or Escape to go back.', - ); - - stdin.write('y'); - await waitForInkInput(); - expect(onExit).toHaveBeenCalledWith({ kind: 'save', rootPageIds: [IDS.engineering] }); - }); - - it('prompts skip-empty confirmation on empty submit and dismisses on cancel', async () => { - const onExit = vi.fn(); - const { stdin, lastFrame } = renderInkTest( - , - ); - - stdin.write('\r'); - await waitForInkInput(); - expect(normalizeFrameWrap(lastFrame())).toContain( - 'Nothing selected. Skip this step? Press Enter to skip or Escape to go back.', - ); - expect(onExit).not.toHaveBeenCalled(); - - stdin.write('n'); - await waitForInkInput(); - expect(lastFrame()).not.toContain('Nothing selected. Skip this step?'); - expect(onExit).not.toHaveBeenCalled(); - - stdin.write('\r'); - await waitForInkInput(); - expect(lastFrame()).toContain('Nothing selected. Skip this step?'); - - stdin.write('\r'); - await waitForInkInput(); - expect(onExit).toHaveBeenCalledWith({ kind: 'quit' }); - }); - - it('renders row-window overflow indicators when the visible list is clipped', async () => { - const onExit = vi.fn(); - const initialState = buildInitialState({ - tree: buildPickerTree(manyPages()), - existingRootPageIds: [], - currentCrawlMode: 'selected_roots', - }); - initialState.expanded = new Set([IDS.engineering]); - const { stdin, lastFrame } = renderInkTest( - , - ); - - expect(lastFrame()).toContain('↓ 4 more'); - - stdin.write('\u001B[B'); - stdin.write('\u001B[B'); - stdin.write('\u001B[B'); - stdin.write('\u001B[B'); - await waitForInkInput(); - - const frame = lastFrame() ?? ''; - expect(frame).toContain('↑ '); - expect(frame).toContain('↓ '); - expect(onExit).not.toHaveBeenCalled(); - }); - - it('returns quit without saving', async () => { - const onExit = vi.fn(); - const { stdin } = renderInkTest( - , - ); - - stdin.write('\u0003'); - await waitForInkInput(); - expect(onExit).toHaveBeenCalledWith({ kind: 'quit' }); - }); -}); - -describe('renderNotionPickerTui', () => { - it('returns the app result from the Ink runtime', async () => { - const io = { - stdin: { isTTY: true, setRawMode: vi.fn() }, - stdout: { isTTY: true, columns: 100, rows: 24, write: vi.fn() }, - stderr: { write: vi.fn() }, - }; - const renderInk = vi.fn((_tree: ReactNode, _options: NotionPickerInkRenderOptions) => fakeInkInstance()); - - await expect( - renderNotionPickerTui( - { - initialState: state(), - connectionId: 'notion-main', - workspaceLabel: 'Design Workspace', - cappedAtCount: null, - currentCrawlMode: 'selected_roots', - }, - io, - { renderInk }, - ), - ).resolves.toEqual({ kind: 'quit' }); - expect(renderInk).toHaveBeenCalledOnce(); - }); - - it('sanitizes render errors and tells the user to use no-input mode', async () => { - expect(sanitizeNotionPickerTuiError(new Error('token=secret https://api.notion.com/v1/search'))).toBe( - '[redacted] [redacted-url]', - ); - }); - - it('falls back to quit with a scripted-mode hint when Ink cannot initialize', async () => { - let stderr = ''; - const io = { - stdin: { isTTY: false, setRawMode: vi.fn() }, - stdout: { isTTY: false, columns: 100, rows: 24, write: vi.fn() }, - stderr: { - write(chunk: string) { - stderr += chunk; - }, - }, - }; - - await expect( - renderNotionPickerTui( - { - initialState: state(), - connectionId: 'notion-main', - workspaceLabel: 'Design Workspace', - cappedAtCount: null, - currentCrawlMode: 'selected_roots', - }, - io, - { - renderInk: vi.fn(() => { - throw new Error('token=secret'); - }), - }, - ), - ).resolves.toEqual({ kind: 'quit' }); - expect(stderr).toContain('Use --no-input --notion-root-page-id for scripted mode'); - expect(stderr).not.toContain('secret'); - }); -}); diff --git a/packages/cli/src/notion-page-picker.test.ts b/packages/cli/src/notion-page-picker.test.ts index 77710716..29f5a352 100644 --- a/packages/cli/src/notion-page-picker.test.ts +++ b/packages/cli/src/notion-page-picker.test.ts @@ -1,4 +1,6 @@ import { describe, expect, it, vi } from 'vitest'; +import type { PickerState } from './tree-picker-state.js'; +import type { TreePickerChrome, TreePickerResult, TreePickerTuiIo } from './tree-picker-tui.js'; import { discoverNotionPickerPages, notionPickerPageFromSearchResult, @@ -6,8 +8,6 @@ import { pickNotionRootPages, resolveNotionWorkspaceLabel, type NotionPickerApi, - type PickerRenderInput, - type PickerRenderResult, } from './notion-page-picker.js'; function makeIo() { @@ -162,20 +162,27 @@ describe('Notion page picker helpers', () => { }); }); +type RenderPickerArgs = [TreePickerChrome, PickerState, TreePickerTuiIo]; + describe('pickNotionRootPages', () => { it('discovers visible pages, warns about stale roots, renders the TUI, and returns selected roots', async () => { const api = fakeNotionApi([ notionPage(PAGE_IDS.engineering, 'Engineering'), notionPage(PAGE_IDS.architecture, 'Architecture', PAGE_IDS.engineering), ]); - const renderPicker = vi.fn(async (input: PickerRenderInput): Promise => { - expect(input.connectionId).toBe('notion-main'); - expect(input.workspaceLabel).toBe('Design Workspace'); - expect(input.currentCrawlMode).toBe('all_accessible'); - expect(input.cappedAtCount).toBeNull(); - expect(input.initialState.preLoadWarnings).toEqual(['1 stored root_page_ids no longer visible']); - return { kind: 'save', rootPageIds: [PAGE_IDS.engineering] }; - }); + const renderPicker = vi.fn( + async (chrome: TreePickerChrome, state: PickerState): Promise => { + expect(chrome.title).toBe('Select Notion pages to ingest'); + expect(chrome.subtitleLines).toEqual(['Workspace: Design Workspace']); + expect(chrome.warningLines ?? []).toEqual([]); + expect(chrome.confirmSaveMessage).toBeTypeOf('function'); + expect(state.requireConfirmOnSave).toBe(true); + expect(state.preLoadWarnings).toEqual([ + '1 stored root_page_ids no longer visible - they will be removed if you save', + ]); + return { kind: 'save', selectedIds: [PAGE_IDS.engineering] }; + }, + ); const io = makeIo(); await expect( @@ -223,7 +230,7 @@ describe('pickNotionRootPages', () => { makeIo().io, { createNotionApi, - renderPicker: vi.fn(async (): Promise => ({ kind: 'quit' })), + renderPicker: vi.fn(async (): Promise => ({ kind: 'quit' })), }, ), ).resolves.toEqual({ kind: 'back' }); @@ -243,11 +250,13 @@ describe('pickNotionRootPages', () => { .mockRejectedValueOnce(new Error('rate limit after first page')), retrieveBotUser: vi.fn(async () => ({ name: 'Notion bot', bot: { workspace_name: 'Design Workspace' } })), }; - let renderInput: PickerRenderInput | undefined; - const renderPicker = vi.fn(async (input: PickerRenderInput): Promise => { - renderInput = input; - return { kind: 'quit' }; - }); + let captured: RenderPickerArgs | undefined; + const renderPicker = vi.fn( + async (chrome: TreePickerChrome, state: PickerState, io: TreePickerTuiIo): Promise => { + captured = [chrome, state, io]; + return { kind: 'quit' }; + }, + ); const io = makeIo(); await expect( @@ -271,11 +280,12 @@ describe('pickNotionRootPages', () => { ).resolves.toEqual({ kind: 'back' }); expect(renderPicker).toHaveBeenCalledOnce(); - if (!renderInput) { + if (!captured) { throw new Error('renderPicker was not called'); } - expect(renderInput.initialState.preLoadWarnings).toEqual(['Notion search stopped early: rate limit after first page']); - expect(renderInput.initialState.tree.map((node) => node.title)).toEqual(['Engineering']); + const [, state] = captured; + expect(state.preLoadWarnings).toEqual(['Notion search stopped early: rate limit after first page']); + expect(state.tree.map((node) => node.title)).toEqual(['Engineering']); expect(io.stderr()).toContain('Notion search stopped early: rate limit after first page'); }); @@ -300,7 +310,7 @@ describe('pickNotionRootPages', () => { }), retrieveBotUser: vi.fn(async () => ({ name: 'Notion bot' })), })), - renderPicker: vi.fn(async (): Promise => ({ kind: 'quit' })), + renderPicker: vi.fn(async (): Promise => ({ kind: 'quit' })), }, ), ).resolves.toEqual({ kind: 'unavailable', message: 'Notion API unavailable' }); diff --git a/packages/cli/src/notion-page-picker.ts b/packages/cli/src/notion-page-picker.ts index 807c0fc0..26e561f5 100644 --- a/packages/cli/src/notion-page-picker.ts +++ b/packages/cli/src/notion-page-picker.ts @@ -3,13 +3,19 @@ import { type NotionApi, type NotionBotInfo, NotionClient } from '@ktx/context/i import type { KtxProjectConnectionConfig } from '@ktx/context/project'; import type { KtxCliIo } from './cli-runtime.js'; import { profileMark } from './startup-profile.js'; -import { buildInitialState, buildPickerTree, type NotionPickerPageInput } from './notion-page-picker-tree.js'; import { - type NotionPickerTuiIo, - type PickerRenderInput, - type PickerRenderResult, - renderNotionPickerTui, -} from './notion-page-picker-tui.js'; + buildInitialState, + buildPickerTree, + flattenSelection, + type PickerState, + type TreePickerNodeInput, +} from './tree-picker-state.js'; +import { + renderTreePickerTui, + type TreePickerChrome, + type TreePickerResult, + type TreePickerTuiIo, +} from './tree-picker-tui.js'; profileMark('module:notion-page-picker'); @@ -19,8 +25,6 @@ export interface PickNotionRootPagesArgs { } export type NotionPickerApi = Pick; -export type { PickerRenderInput, PickerRenderResult }; - export type NotionRootPagePickResult = | { kind: 'selected'; rootPageIds: string[] } | { kind: 'back' } @@ -29,10 +33,16 @@ export type NotionRootPagePickResult = export interface NotionRootPagePickerDeps { env?: Record; createNotionApi?: (authToken: string) => NotionPickerApi; - renderPicker?: (input: PickerRenderInput, io: NotionPickerTuiIo) => Promise; + renderPicker?: ( + chrome: TreePickerChrome, + initialState: PickerState, + io: TreePickerTuiIo, + ) => Promise; } const NOTION_PICKER_PAGE_CAP = 5000; +const NOTION_SCRIPTED_MODE_HINT = + 'Notion picker requires a TTY. Use --no-input --notion-root-page-id for scripted mode.'; function assertSafeNotionPickerConnectionId(connectionId: string): void { if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(connectionId)) { @@ -50,6 +60,14 @@ export function normalizeNotionPageId(value: string): string { return `${lower.slice(0, 8)}-${lower.slice(8, 12)}-${lower.slice(12, 16)}-${lower.slice(16, 20)}-${lower.slice(20)}`; } +function tryNormalizeNotionPageId(value: string): string | null { + try { + return normalizeNotionPageId(value); + } catch { + return null; + } +} + function recordValue(value: unknown): Record | null { return typeof value === 'object' && value !== null && !Array.isArray(value) ? (value as Record) @@ -88,7 +106,7 @@ function extractParentPageId(page: Record): string | null { return normalizeNotionPageId(parent.page_id); } -export function notionPickerPageFromSearchResult(result: Record): NotionPickerPageInput { +export function notionPickerPageFromSearchResult(result: Record): TreePickerNodeInput { const id = typeof result.id === 'string' ? normalizeNotionPageId(result.id) : ''; if (!id) { throw new Error('Notion page search result is missing id'); @@ -104,9 +122,9 @@ export function notionPickerPageFromSearchResult(result: Record export async function discoverNotionPickerPages( api: NotionPickerApi, options: { cap?: number } = {}, -): Promise<{ pages: NotionPickerPageInput[]; cappedAtCount: number | null; warnings: string[] }> { +): Promise<{ pages: TreePickerNodeInput[]; cappedAtCount: number | null; warnings: string[] }> { const cap = options.cap ?? NOTION_PICKER_PAGE_CAP; - const pages: NotionPickerPageInput[] = []; + const pages: TreePickerNodeInput[] = []; const warnings: string[] = []; let cursor: string | null | undefined = null; @@ -171,6 +189,33 @@ function notionCrawlMode(connection: KtxProjectConnectionConfig): 'all_accessibl return connection.crawl_mode === 'all_accessible' ? 'all_accessible' : 'selected_roots'; } +function selectedPageCountText(count: number): string { + return `${count} selected ${count === 1 ? 'page' : 'pages'}`; +} + +function notionChrome(args: { + workspaceLabel: string; + cappedAtCount: number | null; + currentCrawlMode: 'all_accessible' | 'selected_roots'; +}): TreePickerChrome { + const warningLines: string[] = []; + if (args.cappedAtCount) { + warningLines.push(`${args.cappedAtCount}-page cap reached - some pages not shown`); + } + return { + title: 'Select Notion pages to ingest', + subtitleLines: [`Workspace: ${args.workspaceLabel}`], + warningLines, + confirmSaveMessage: + args.currentCrawlMode === 'all_accessible' + ? (state) => + `Switch crawl_mode from all_accessible to selected_roots? Will limit ingest to ${selectedPageCountText( + flattenSelection(state.checked, state.byId).length, + )}. Press Enter to confirm or Escape to go back.` + : undefined, + }; +} + export async function pickNotionRootPages( args: PickNotionRootPagesArgs, io: KtxCliIo = process, @@ -190,10 +235,14 @@ export async function pickNotionRootPages( const api = deps.createNotionApi ? deps.createNotionApi(authToken) : new NotionClient(authToken); const discovery = await discoverNotionPickerPages(api); const tree = buildPickerTree(discovery.pages); + const normalizedExistingIds = stringArray(args.connection.root_page_ids) + .map((raw) => tryNormalizeNotionPageId(raw)) + .filter((id): id is string => id !== null); const initialState = buildInitialState({ tree, - existingRootPageIds: stringArray(args.connection.root_page_ids), - currentCrawlMode: crawlMode, + existingSelectedIds: normalizedExistingIds, + requireConfirmOnSave: crawlMode === 'all_accessible', + staleWarning: (count) => `${count} stored root_page_ids no longer visible - they will be removed if you save`, }); const preLoadWarnings = [...discovery.warnings, ...initialState.preLoadWarnings]; const renderState = @@ -207,23 +256,25 @@ export async function pickNotionRootPages( io.stderr.write(`${warning}\n`); } const workspaceLabel = await resolveNotionWorkspaceLabel(api, args.connectionId); - const result = await (deps.renderPicker ?? renderNotionPickerTui)( - { - initialState: renderState, - connectionId: args.connectionId, - workspaceLabel, - cappedAtCount: discovery.cappedAtCount, - currentCrawlMode: crawlMode, - }, - io as NotionPickerTuiIo, - ); + const chrome = notionChrome({ + workspaceLabel, + cappedAtCount: discovery.cappedAtCount, + currentCrawlMode: crawlMode, + }); + const renderPicker = + deps.renderPicker ?? + ((chromeArg, state, ioArg) => + renderTreePickerTui({ chrome: chromeArg, initialState: state }, ioArg, { + scriptedModeHint: NOTION_SCRIPTED_MODE_HINT, + })); + const result = await renderPicker(chrome, renderState, io as TreePickerTuiIo); if (result.kind === 'quit') { return { kind: 'back' }; } - if (result.rootPageIds.length === 0) { + if (result.selectedIds.length === 0) { return { kind: 'unavailable', message: 'Notion picker did not return any selected pages.' }; } - return { kind: 'selected', rootPageIds: result.rootPageIds }; + return { kind: 'selected', rootPageIds: result.selectedIds }; } catch (error) { return { kind: 'unavailable', message: error instanceof Error ? error.message : String(error) }; } diff --git a/packages/cli/src/setup-databases.test.ts b/packages/cli/src/setup-databases.test.ts index fa4ca3f2..d3a55fba 100644 --- a/packages/cli/src/setup-databases.test.ts +++ b/packages/cli/src/setup-databases.test.ts @@ -5,10 +5,15 @@ import { initKtxProject, parseKtxProjectConfig, readKtxSetupState, writeKtxSetup import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { type KtxSetupDatabaseDriver, + type KtxSetupDatabasesDeps, type KtxSetupDatabasesPromptAdapter, runKtxSetupDatabasesStep, } from './setup-databases.js'; import type { KtxCliIo } from './cli-runtime.js'; +import type { + DatabaseScopePickResult, + PickDatabaseScopeArgs, +} from './database-tree-picker.js'; function makeIo() { let stdout = ''; @@ -32,6 +37,43 @@ function makeIo() { }; } +type ScopePick = + | 'back' + | 'enable-all' + | { schemas: string[]; tables: string[] }; + +interface PickerStubs { + pickDatabaseScope: KtxSetupDatabasesDeps['pickDatabaseScope']; + scopeCalls: PickDatabaseScopeArgs[]; +} + +function makePickerStubs(options: { scopes?: ScopePick[] } = {}): PickerStubs { + const queue: ScopePick[] = [...(options.scopes ?? [])]; + const scopeCalls: PickDatabaseScopeArgs[] = []; + return { + scopeCalls, + pickDatabaseScope: vi.fn(async (args: PickDatabaseScopeArgs): Promise => { + scopeCalls.push(args); + const next = queue.shift(); + if (next === undefined || next === 'enable-all') { + const enabledTables = args.discovered.map((t) => `${t.schema}.${t.name}`); + const activeSchemas = args.supportsSchemaScope + ? Array.from(new Set(args.discovered.map((t) => t.schema))) + : []; + return { kind: 'selected', activeSchemas, enabledTables }; + } + if (next === 'back') { + return { kind: 'back' }; + } + return { + kind: 'selected', + activeSchemas: args.supportsSchemaScope ? next.schemas : [], + enabledTables: next.tables, + }; + }), + }; +} + function makePromptAdapter(options: { multiselectValues?: string[][]; selectValues?: string[]; @@ -819,7 +861,6 @@ describe('setup databases step', () => { await writeKtxSetupState(tempDir, { completed_steps: ['databases'] }); const prompts = makePromptAdapter({ textValues: ['env:DATABASE_URL'], - multiselectValues: [['analytics']], }); let primaryMenuCount = 0; vi.mocked(prompts.select).mockImplementation(async (options) => { @@ -835,11 +876,21 @@ describe('setup databases step', () => { const scanConnection = vi.fn(async () => 0); const listSchemas = vi.fn(async () => ['analytics', 'public']); const listTables = vi.fn(async () => [{ schema: 'analytics', name: 'customers', kind: 'table' as const }]); + const pickers = makePickerStubs({ + scopes: [{ schemas: ['analytics'], tables: ['analytics.customers'] }], + }); const result = await runKtxSetupDatabasesStep( { projectDir: tempDir, inputMode: 'auto', skipDatabases: false, databaseSchemas: [] }, makeIo().io, - { prompts, testConnection, scanConnection, listSchemas, listTables }, + { + prompts, + testConnection, + scanConnection, + listSchemas, + listTables, + pickDatabaseScope: pickers.pickDatabaseScope, + }, ); expect(result).toEqual({ status: 'ready', projectDir: tempDir, connectionIds: ['warehouse'] }); @@ -848,7 +899,7 @@ describe('setup databases step', () => { placeholder: 'env:DATABASE_URL', initialValue: 'env:DATABASE_URL', }); - expect(listTables).toHaveBeenCalledWith(tempDir, 'warehouse'); + expect(listTables).toHaveBeenCalledWith(tempDir, 'warehouse', ['analytics', 'public']); expect(testConnection).toHaveBeenCalledWith(tempDir, 'warehouse', expect.anything()); expect(scanConnection).toHaveBeenCalledWith(tempDir, 'warehouse', expect.anything()); const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); @@ -882,7 +933,6 @@ describe('setup databases step', () => { await writeKtxSetupState(tempDir, { completed_steps: ['databases'] }); const prompts = makePromptAdapter({ textValues: ['env:DATABASE_URL'], - multiselectValues: [['public'], ['public.customers', 'public.orders']], }); let primaryMenuCount = 0; vi.mocked(prompts.select).mockImplementation(async (options) => { @@ -892,7 +942,6 @@ describe('setup databases step', () => { } if (options.message === 'Primary source to edit') return 'warehouse'; if (options.message === 'How do you want to connect to PostgreSQL?') return 'url'; - if (options.message.startsWith('Tables found in selected schemas')) return 'customize'; return 'back'; }); const listSchemas = vi.fn(async () => ['orbit_analytics', 'orbit_raw', 'public']); @@ -901,6 +950,9 @@ describe('setup databases step', () => { { schema: 'public', name: 'orders', kind: 'table' as const }, { schema: 'public', name: 'products', kind: 'table' as const }, ]); + const pickers = makePickerStubs({ + scopes: [{ schemas: ['public'], tables: ['public.customers', 'public.orders'] }], + }); const result = await runKtxSetupDatabasesStep( { projectDir: tempDir, inputMode: 'auto', skipDatabases: false, databaseSchemas: [] }, @@ -911,29 +963,17 @@ describe('setup databases step', () => { scanConnection: vi.fn(async () => 0), listSchemas, listTables, + pickDatabaseScope: pickers.pickDatabaseScope, }, ); expect(result).toEqual({ status: 'ready', projectDir: tempDir, connectionIds: ['warehouse'] }); - expect(prompts.multiselect).toHaveBeenNthCalledWith(1, { - message: expect.stringContaining('PostgreSQL schemas to scan'), - options: [ - { value: 'orbit_analytics', label: 'orbit_analytics' }, - { value: 'orbit_raw', label: 'orbit_raw' }, - { value: 'public', label: 'public' }, - ], - initialValues: ['public'], - required: true, - }); - expect(prompts.multiselect).toHaveBeenNthCalledWith(2, { - message: expect.stringContaining('Tables to enable for warehouse'), - options: [ - { value: 'public.customers', label: 'public.customers' }, - { value: 'public.orders', label: 'public.orders' }, - { value: 'public.products', label: 'public.products' }, - ], - initialValues: ['public.customers', 'public.orders'], - required: true, + expect(pickers.scopeCalls).toHaveLength(1); + expect(pickers.scopeCalls[0]).toMatchObject({ + connectionId: 'warehouse', + schemaNoun: 'schema', + supportsSchemaScope: true, + existing: { enabledTables: ['public.customers', 'public.orders'] }, }); const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); expect(config.connections.warehouse).toMatchObject({ @@ -965,7 +1005,6 @@ describe('setup databases step', () => { await writeKtxSetupState(tempDir, { completed_steps: ['databases'] }); const prompts = makePromptAdapter({ textValues: ['env:DATABASE_URL'], - multiselectValues: [['back']], }); let primaryMenuCount = 0; vi.mocked(prompts.select).mockImplementation(async (options) => { @@ -980,19 +1019,29 @@ describe('setup databases step', () => { const testConnection = vi.fn(async () => 0); const scanConnection = vi.fn(async () => 0); const listSchemas = vi.fn(async () => ['analytics', 'public']); - const listTables = vi.fn(async () => [{ schema: 'analytics', name: 'customers', kind: 'table' as const }]); + const listTables = vi.fn(async () => [ + { schema: 'analytics', name: 'customers', kind: 'table' as const }, + { schema: 'public', name: 'orders', kind: 'table' as const }, + ]); + const pickers = makePickerStubs({ scopes: ['back'] }); const result = await runKtxSetupDatabasesStep( { projectDir: tempDir, inputMode: 'auto', skipDatabases: false, databaseSchemas: [] }, makeIo().io, - { prompts, testConnection, scanConnection, listSchemas, listTables }, + { + prompts, + testConnection, + scanConnection, + listSchemas, + listTables, + pickDatabaseScope: pickers.pickDatabaseScope, + }, ); expect(result).toEqual({ status: 'ready', projectDir: tempDir, connectionIds: ['warehouse'] }); expect(primaryMenuCount).toBe(2); expect(testConnection).toHaveBeenCalledWith(tempDir, 'warehouse', expect.anything()); expect(scanConnection).not.toHaveBeenCalled(); - expect(listTables).not.toHaveBeenCalled(); const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); expect(config.connections.warehouse).toMatchObject({ url: 'env:DATABASE_URL', @@ -1031,7 +1080,6 @@ describe('setup databases step', () => { } if (options.message === 'Primary source to edit') return 'warehouse'; if (options.message === 'How do you want to connect to PostgreSQL?') return 'url'; - if (options.message.startsWith('Tables found in selected schemas')) return 'back'; return 'back'; }); const testConnection = vi.fn(async () => 0); @@ -1041,16 +1089,24 @@ describe('setup databases step', () => { { schema: 'public', name: 'customers', kind: 'table' as const }, { schema: 'public', name: 'orders', kind: 'table' as const }, ]); + const pickers = makePickerStubs({ scopes: ['back'] }); const result = await runKtxSetupDatabasesStep( { projectDir: tempDir, inputMode: 'auto', skipDatabases: false, databaseSchemas: [] }, makeIo().io, - { prompts, testConnection, scanConnection, listSchemas, listTables }, + { + prompts, + testConnection, + scanConnection, + listSchemas, + listTables, + pickDatabaseScope: pickers.pickDatabaseScope, + }, ); expect(result).toEqual({ status: 'ready', projectDir: tempDir, connectionIds: ['warehouse'] }); expect(primaryMenuCount).toBe(2); - expect(listTables).toHaveBeenCalledWith(tempDir, 'warehouse'); + expect(listTables).toHaveBeenCalledWith(tempDir, 'warehouse', ['public']); expect(scanConnection).not.toHaveBeenCalled(); const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); expect(config.connections.warehouse).toMatchObject({ @@ -1083,19 +1139,18 @@ describe('setup databases step', () => { await writeKtxSetupState(tempDir, { completed_steps: ['databases'] }); const prompts = makePromptAdapter({ textValues: ['env:DATABASE_URL'], - multiselectValues: [['public']], }); vi.mocked(prompts.select).mockImplementation(async (options) => { if (options.message === 'Primary sources already configured: warehouse\nWhat would you like to do?') return 'edit'; if (options.message === 'Primary source to edit') return 'warehouse'; if (options.message === 'How do you want to connect to PostgreSQL?') return 'url'; - if (options.message.startsWith('Tables found in selected schemas')) return 'all'; return 'back'; }); const listTables = vi.fn(async () => [ { schema: 'public', name: 'customers', kind: 'table' as const }, { schema: 'public', name: 'orders', kind: 'table' as const }, ]); + const pickers = makePickerStubs({ scopes: ['enable-all'] }); const result = await runKtxSetupDatabasesStep( { projectDir: tempDir, inputMode: 'auto', skipDatabases: false, databaseSchemas: [] }, @@ -1105,6 +1160,7 @@ describe('setup databases step', () => { testConnection: vi.fn(async () => 0), scanConnection: vi.fn(async () => 1), listTables, + pickDatabaseScope: pickers.pickDatabaseScope, }, ); @@ -1390,7 +1446,6 @@ describe('setup databases step', () => { const prompts = makePromptAdapter({ selectValues: ['url'], textValues: ['', 'env:DATABASE_URL'], - multiselectValues: [['orbit_analytics', 'orbit_raw']], }); const testConnection = vi.fn(async () => 0); const scanConnection = vi.fn(async asyncScanProjectDir => { @@ -1401,6 +1456,19 @@ describe('setup databases step', () => { return 0; }); const listSchemas = vi.fn(async () => ['orbit_analytics', 'orbit_raw', 'public']); + const listTables = vi.fn(async () => [ + { schema: 'orbit_analytics', name: 'events', kind: 'table' as const }, + { schema: 'orbit_raw', name: 'inputs', kind: 'table' as const }, + { schema: 'public', name: 'misc', kind: 'table' as const }, + ]); + const pickers = makePickerStubs({ + scopes: [ + { + schemas: ['orbit_analytics', 'orbit_raw'], + tables: ['orbit_analytics.events', 'orbit_raw.inputs'], + }, + ], + }); const result = await runKtxSetupDatabasesStep( { @@ -1411,20 +1479,24 @@ describe('setup databases step', () => { skipDatabases: false, }, io.io, - { prompts, testConnection, scanConnection, listSchemas }, + { + prompts, + testConnection, + scanConnection, + listSchemas, + listTables, + pickDatabaseScope: pickers.pickDatabaseScope, + }, ); expect(result.status).toBe('ready'); expect(listSchemas).toHaveBeenCalledWith(tempDir, 'postgres-warehouse'); - expect(prompts.multiselect).toHaveBeenCalledWith({ - message: expect.stringContaining('PostgreSQL schemas to scan'), - options: [ - { value: 'orbit_analytics', label: 'orbit_analytics' }, - { value: 'orbit_raw', label: 'orbit_raw' }, - { value: 'public', label: 'public' }, - ], - initialValues: ['orbit_analytics', 'orbit_raw'], - required: true, + expect(pickers.scopeCalls).toHaveLength(1); + expect(pickers.scopeCalls[0]).toMatchObject({ + connectionId: 'postgres-warehouse', + schemaNoun: 'schema', + schemaNounPlural: 'schemas', + defaultSchemas: ['orbit_analytics', 'orbit_raw'], }); const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); expect(config.connections['postgres-warehouse']).toMatchObject({ diff --git a/packages/cli/src/setup-databases.ts b/packages/cli/src/setup-databases.ts index 9db80689..c21ab6d1 100644 --- a/packages/cli/src/setup-databases.ts +++ b/packages/cli/src/setup-databases.ts @@ -14,6 +14,11 @@ import { import type { KtxTableListEntry } from '@ktx/context/scan'; import type { KtxCliIo } from './cli-runtime.js'; import { runKtxConnection } from './connection.js'; +import { + pickDatabaseScope as defaultPickDatabaseScope, + type DatabaseScopePickResult, + type PickDatabaseScopeArgs, +} from './database-tree-picker.js'; import { withMultiselectNavigation, withTextInputNavigation } from './prompt-navigation.js'; import { runKtxScan } from './scan.js'; import { writeProjectLocalSecretReference } from './setup-secrets.js'; @@ -90,7 +95,8 @@ export interface KtxSetupDatabasesDeps { scanConnection?: (projectDir: string, connectionId: string, io: KtxCliIo) => Promise; rebuildNativeSqlite?: (io: KtxCliIo) => Promise; listSchemas?: (projectDir: string, connectionId: string) => Promise; - listTables?: (projectDir: string, connectionId: string) => Promise; + listTables?: (projectDir: string, connectionId: string, schemas?: string[]) => Promise; + pickDatabaseScope?: (args: PickDatabaseScopeArgs, io: KtxCliIo) => Promise; historicSqlProbe?: KtxSetupHistoricSqlProbe; } @@ -363,11 +369,15 @@ function configuredSchemas(connection: KtxProjectConnectionConfig | undefined, d return values.length > 0 ? values : undefined; } -async function defaultListTables(projectDir: string, connectionId: string): Promise { +async function defaultListTables( + projectDir: string, + connectionId: string, + schemasOverride?: string[], +): Promise { const project = await loadKtxProject({ projectDir }); const connection = project.config.connections[connectionId]; const driver = normalizeDriver(connection?.driver); - const schemas = driver ? configuredSchemas(connection, driver) : undefined; + const schemas = schemasOverride ?? (driver ? configuredSchemas(connection, driver) : undefined); if (driver === 'postgres') { const { KtxPostgresScanConnector, isKtxPostgresConnectionConfig } = await import('@ktx/connector-postgres'); @@ -1271,145 +1281,98 @@ async function writeScopeConfig(input: { }); } -async function clearScopeConfig(projectDir: string, connectionId: string): Promise { - const project = await loadKtxProject({ projectDir }); - const connection = project.config.connections[connectionId]; - if (!connection) return; - const driver = normalizeDriver(connection.driver); - if (!driver) return; - const spec = SCOPE_DISCOVERY_SPECS[driver]; - if (!spec) return; - const cleaned = Object.fromEntries( - Object.entries(connection).filter( - ([key]) => key !== spec.configArrayField && key !== spec.configSingleField && key !== 'enabled_tables', - ), - ) as KtxProjectConnectionConfig; - await writeConnectionConfig({ projectDir, connectionId, connection: cleaned }); -} - -async function maybeConfigureSchemaScope(input: { +async function maybeConfigureDatabaseScope(input: { projectDir: string; connectionId: string; args: KtxSetupDatabasesArgs; - prompts: KtxSetupDatabasesPromptAdapter; deps: KtxSetupDatabasesDeps; io: KtxCliIo; forcePrompt?: boolean; -}): Promise { - const project = await loadKtxProject({ projectDir: input.projectDir }); - const connection = project.config.connections[input.connectionId]; - const driver = normalizeDriver(connection?.driver); - if (!driver) return 'ready'; - - const spec = SCOPE_DISCOVERY_SPECS[driver]; - if (!spec) return 'ready'; - - const arrayVal = connection?.[spec.configArrayField]; - if (Array.isArray(arrayVal) && arrayVal.length > 0 && input.forcePrompt !== true) { - return 'ready'; - } - - if (input.args.databaseSchemas.length > 0) { - await writeScopeConfig({ - projectDir: input.projectDir, - connectionId: input.connectionId, - values: input.args.databaseSchemas, - spec, - }); - return 'ready'; - } - - writeSetupSection(input.io, `Discovering ${spec.promptLabel.toLowerCase()}`, [ - `Connecting to ${input.connectionId}…`, - ]); - - let discovered: string[]; - try { - discovered = unique( - await (input.deps.listSchemas ?? defaultListSchemas)(input.projectDir, input.connectionId), - ); - } catch (error) { - const detail = error instanceof Error ? error.message : String(error); - input.io.stderr.write( - input.forcePrompt === true - ? `Could not discover ${spec.promptLabel.toLowerCase()} for ${input.connectionId}; edit was not saved. ` + - `Pass --database-schema to set it explicitly. ${detail}\n` - : `Could not discover ${spec.promptLabel.toLowerCase()} for ${input.connectionId}; continuing with existing ${spec.noun} scope. ` + - `Pass --database-schema to set it explicitly. ${detail}\n`, - ); - return input.forcePrompt === true ? 'failed' : 'ready'; - } - if (discovered.length === 0) { - return 'ready'; - } - - let selected: string[]; - if (input.args.inputMode === 'disabled' || discovered.length === 1) { - const preconfigured = configuredScopeValues(connection, spec).filter((v) => discovered.includes(v)); - selected = preconfigured.length > 0 ? preconfigured : discovered; - } else { - const preconfigured = configuredScopeValues(connection, spec).filter((v) => discovered.includes(v)); - const initialValues = preconfigured.length > 0 ? preconfigured : spec.defaultSelection(discovered); - const choices = await input.prompts.multiselect({ - message: withMultiselectNavigation( - `${spec.promptLabel} to scan\n` + - `KTX found multiple ${spec.nounPlural}. Select every ${spec.noun} agents should use.`, - ), - options: discovered.map((v) => ({ value: v, label: v })), - initialValues, - required: true, - }); - if (choices.includes('back')) { - return 'back'; - } - selected = choices.length > 0 ? choices : initialValues; - } - - await writeScopeConfig({ - projectDir: input.projectDir, - connectionId: input.connectionId, - values: selected, - spec, - }); - const capitalNounPlural = spec.nounPlural[0]!.toUpperCase() + spec.nounPlural.slice(1); - writeSetupSection(input.io, `${capitalNounPlural} saved for ${input.connectionId}`, [ - `✓ ${selected.join(', ')}`, - ]); - return 'ready'; -} - -async function maybeConfigureTableScope(input: { - projectDir: string; - connectionId: string; - args: KtxSetupDatabasesArgs; - prompts: KtxSetupDatabasesPromptAdapter; - io: KtxCliIo; - deps: KtxSetupDatabasesDeps; - forcePrompt?: boolean; }): Promise { const project = await loadKtxProject({ projectDir: input.projectDir }); const connection = project.config.connections[input.connectionId]; const driver = normalizeDriver(connection?.driver); if (!driver || driver === 'sqlite') return 'ready'; + const spec = SCOPE_DISCOVERY_SPECS[driver]; const existingTables = connection?.enabled_tables; - if (Array.isArray(existingTables) && existingTables.length > 0 && input.forcePrompt !== true) { + const hasExistingTables = Array.isArray(existingTables) && existingTables.length > 0; + const existingScope = spec ? configuredScopeValues(connection, spec) : []; + const hasExistingScope = !spec || existingScope.length > 0; + + if (hasExistingTables && hasExistingScope && input.forcePrompt !== true) { return 'ready'; } + const cliSchemas = input.args.databaseSchemas; + if (input.args.inputMode === 'disabled') { + if (spec) { + let scopeToWrite: string[] = cliSchemas; + if (scopeToWrite.length === 0) { + try { + scopeToWrite = unique( + await (input.deps.listSchemas ?? defaultListSchemas)(input.projectDir, input.connectionId), + ); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + input.io.stderr.write( + `Could not discover ${spec.promptLabel.toLowerCase()} for ${input.connectionId}; ${detail}\n`, + ); + return 'ready'; + } + } + if (scopeToWrite.length > 0) { + await writeScopeConfig({ + projectDir: input.projectDir, + connectionId: input.connectionId, + values: scopeToWrite, + spec, + }); + const capitalNounPlural = spec.nounPlural[0]!.toUpperCase() + spec.nounPlural.slice(1); + writeSetupSection(input.io, `${capitalNounPlural} saved for ${input.connectionId}`, [ + `✓ ${scopeToWrite.join(', ')}`, + ]); + } + } return 'ready'; } + if (spec && cliSchemas.length > 0) { + await writeScopeConfig({ + projectDir: input.projectDir, + connectionId: input.connectionId, + values: cliSchemas, + spec, + }); + } + writeSetupSection(input.io, 'Discovering tables', [ `Connecting to ${input.connectionId}…`, ]); + const schemasFilter = await (async (): Promise => { + if (cliSchemas.length > 0) return cliSchemas; + if (!spec) return []; + try { + return unique( + await (input.deps.listSchemas ?? defaultListSchemas)(input.projectDir, input.connectionId), + ); + } catch (error) { + const detail = error instanceof Error ? error.message : String(error); + input.io.stderr.write( + `Could not discover ${spec.promptLabel.toLowerCase()} for ${input.connectionId}; ${detail}\n`, + ); + return []; + } + })(); + let discovered: KtxTableListEntry[]; try { discovered = await (input.deps.listTables ?? defaultListTables)( input.projectDir, input.connectionId, + schemasFilter.length > 0 ? schemasFilter : undefined, ); } catch (error) { const detail = error instanceof Error ? error.message : String(error); @@ -1429,84 +1392,72 @@ async function maybeConfigureTableScope(input: { } const allQualified = discovered.map((t) => `${t.schema}.${t.name}`); + const schemasInDiscovery = unique(discovered.map((t) => t.schema)); + + const defaultSchemas = (() => { + if (cliSchemas.length > 0) return cliSchemas; + if (!spec) return schemasInDiscovery; + return spec.defaultSelection(schemasInDiscovery); + })(); + + const existingEnabled = + hasExistingTables && input.forcePrompt === true + ? (existingTables ?? []).filter( + (table): table is string => typeof table === 'string' && allQualified.includes(table), + ) + : []; + + let activeSchemas: string[]; + let enabledTables: string[]; if (discovered.length === 1) { - await writeConnectionConfig({ - projectDir: input.projectDir, - connectionId: input.connectionId, - connection: { ...connection!, enabled_tables: allQualified }, - }); - writeSetupSection(input.io, `Tables enabled for ${input.connectionId}`, [ - `✓ ${allQualified[0]}`, - ]); - return 'ready'; - } - - const bySchema = new Map(); - for (const entry of discovered) { - const existing = bySchema.get(entry.schema) ?? []; - existing.push(entry); - bySchema.set(entry.schema, existing); - } - const schemaList = [...bySchema.keys()].sort(); - const schemaSummary = schemaList.map((s) => `${s} (${bySchema.get(s)!.length})`).join(', '); - - let selected: string[] | null = null; - - while (selected === null) { - const action = await input.prompts.select({ - message: `Tables found in selected schemas\n` + - `${discovered.length} tables across ${schemaList.length} ${schemaList.length === 1 ? 'schema' : 'schemas'}: ${schemaSummary}`, - options: [ - { value: 'all', label: 'Enable all tables' }, - { value: 'customize', label: 'Customize which tables to enable' }, - { value: 'back', label: 'Back' }, - ], - }); - - if (action === 'back') { + enabledTables = allQualified; + activeSchemas = spec ? schemasInDiscovery : []; + } else { + const pickResult = await (input.deps.pickDatabaseScope ?? defaultPickDatabaseScope)( + { + connectionId: input.connectionId, + schemaNoun: spec?.noun ?? 'schema', + schemaNounPlural: spec?.nounPlural ?? 'schemas', + discovered, + existing: { enabledTables: existingEnabled }, + defaultSchemas, + supportsSchemaScope: spec !== undefined, + }, + input.io, + ); + if (pickResult.kind === 'back') { return 'back'; } - - if (action === 'all') { - selected = allQualified; - } else { - const choices = await input.prompts.multiselect({ - message: withMultiselectNavigation( - `Tables to enable for ${input.connectionId}\n` + - `Deselect any tables agents should not use.`, - ), - options: discovered.map((t) => { - const qualified = `${t.schema}.${t.name}`; - const suffix = t.kind === 'view' ? ' (view)' : ''; - return { value: qualified, label: `${qualified}${suffix}` }; - }), - initialValues: - Array.isArray(existingTables) && input.forcePrompt === true - ? existingTables.filter((table): table is string => typeof table === 'string' && allQualified.includes(table)) - : allQualified, - required: true, - }); - - if (choices.includes('back')) { - continue; - } - if (choices.length === 0) { - input.io.stdout.write('│ KTX needs at least one table enabled. Select a table or press Escape to go back.\n'); - continue; - } - selected = choices; - } + enabledTables = pickResult.enabledTables; + activeSchemas = pickResult.activeSchemas; } + if (spec) { + await writeScopeConfig({ + projectDir: input.projectDir, + connectionId: input.connectionId, + values: activeSchemas, + spec, + }); + } + const refreshedProject = await loadKtxProject({ projectDir: input.projectDir }); + const currentConnection = refreshedProject.config.connections[input.connectionId]; + if (!currentConnection) return 'ready'; await writeConnectionConfig({ projectDir: input.projectDir, connectionId: input.connectionId, - connection: { ...connection!, enabled_tables: selected }, + connection: { ...currentConnection, enabled_tables: enabledTables }, }); + if (spec && activeSchemas.length > 0) { + const capitalNounPlural = spec.nounPlural[0]!.toUpperCase() + spec.nounPlural.slice(1); + writeSetupSection(input.io, `${capitalNounPlural} saved for ${input.connectionId}`, [ + `✓ ${activeSchemas.join(', ')}`, + ]); + } writeSetupSection(input.io, `Tables enabled for ${input.connectionId}`, [ - `✓ ${selected.length}/${discovered.length} tables enabled`, + `✓ ${enabledTables.length}/${discovered.length} tables enabled`, ]); return 'ready'; } @@ -1638,26 +1589,9 @@ async function validateAndScanConnection(input: { const testLines = ['✓ Connection test passed', `Driver: ${driverDisplay}`]; writeSetupSection(input.io, `Testing ${input.connectionId}`, testLines); - while (true) { - const schemaStatus = await maybeConfigureSchemaScope({ ...input, forcePrompt: input.forceScopeAndTables }); - if (schemaStatus !== 'ready') { - return schemaStatus; - } - - const tableStatus = await maybeConfigureTableScope({ ...input, forcePrompt: input.forceScopeAndTables }); - if (tableStatus === 'ready') { - break; - } - - if (input.forceScopeAndTables) { - return tableStatus; - } - - if (tableStatus === 'failed') { - return 'failed'; - } - - await clearScopeConfig(input.projectDir, input.connectionId); + const scopeStatus = await maybeConfigureDatabaseScope({ ...input, forcePrompt: input.forceScopeAndTables }); + if (scopeStatus !== 'ready') { + return scopeStatus; } await maybeRunHistoricSqlSetupProbe({ diff --git a/packages/cli/src/notion-page-picker-tree.test.ts b/packages/cli/src/tree-picker-state.test.ts similarity index 78% rename from packages/cli/src/notion-page-picker-tree.test.ts rename to packages/cli/src/tree-picker-state.test.ts index 58e8c7ca..52e63f3d 100644 --- a/packages/cli/src/notion-page-picker-tree.test.ts +++ b/packages/cli/src/tree-picker-state.test.ts @@ -12,8 +12,8 @@ import { selectNone, toggleChecked, visibleNodeIds, - type NotionPickerPageInput, -} from './notion-page-picker-tree.js'; + type TreePickerNodeInput, +} from './tree-picker-state.js'; const IDS = { engineering: '11111111-1111-1111-1111-111111111111', @@ -27,7 +27,7 @@ const IDS = { cycleB: '99999999-9999-9999-9999-999999999999', }; -function pages(): NotionPickerPageInput[] { +function pages(): TreePickerNodeInput[] { return [ { id: IDS.marketing, title: 'Marketing', archived: false, parentId: null }, { id: IDS.onboarding, title: 'Onboarding', archived: false, parentId: IDS.engineering }, @@ -43,7 +43,7 @@ function pages(): NotionPickerPageInput[] { } describe('buildPickerTree', () => { - it('deduplicates pages, sorts siblings, preserves archived flags, roots orphans, and breaks cycles', () => { + it('deduplicates nodes, sorts siblings, preserves archived flags, roots orphans, and breaks cycles', () => { const tree = buildPickerTree(pages()); const byId = new Map(tree.map((node) => [node.id, node])); @@ -89,8 +89,7 @@ describe('selection invariants', () => { it('checking a parent locks descendants and keeps checked ids minimal', () => { const state = buildInitialState({ tree: buildPickerTree(pages()), - existingRootPageIds: [], - currentCrawlMode: 'selected_roots', + existingSelectedIds: [], }); const checkedParent = toggleChecked(state, IDS.engineering, 1000); @@ -112,15 +111,11 @@ describe('selection invariants', () => { expect(canToggle(IDS.architecture, uncheckedParent)).toEqual({ ok: true }); }); - it('normalizes stored roots, reports stale roots, expands checked ancestors, and flattens descendants', () => { + it('reports stale stored ids via the caller-supplied warning, expands checked ancestors, and flattens descendants', () => { const state = buildInitialState({ tree: buildPickerTree(pages()), - existingRootPageIds: [ - IDS.engineering.replaceAll('-', ''), - IDS.architecture, - 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa', - ], - currentCrawlMode: 'selected_roots', + existingSelectedIds: [IDS.engineering, IDS.architecture, 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa'], + staleWarning: (staleCount) => `${staleCount} stored root_page_ids no longer visible`, }); expect([...state.checked]).toEqual([IDS.engineering]); @@ -129,14 +124,21 @@ describe('selection invariants', () => { expect(state.preLoadWarnings).toEqual(['1 stored root_page_ids no longer visible']); expect(flattenSelection(new Set([IDS.engineering, IDS.architecture]), state.byId)).toEqual([IDS.engineering]); }); + + it('falls back to a generic stale warning when no warning factory is supplied', () => { + const state = buildInitialState({ + tree: buildPickerTree(pages()), + existingSelectedIds: ['aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa'], + }); + expect(state.preLoadWarnings).toEqual(['1 stored selections no longer visible']); + }); }); describe('search and cursor movement', () => { it('filters by title and path while deriving auto-expanded ancestors', () => { const state = buildInitialState({ tree: buildPickerTree(pages()), - existingRootPageIds: [], - currentCrawlMode: 'selected_roots', + existingSelectedIds: [], }); const searching = { ...state, @@ -153,8 +155,7 @@ describe('search and cursor movement', () => { it('moves the cursor through visible nodes and implements left/right tree semantics', () => { const state = buildInitialState({ tree: buildPickerTree(pages()), - existingRootPageIds: [], - currentCrawlMode: 'selected_roots', + existingSelectedIds: [], }); const atEngineering = { @@ -175,8 +176,7 @@ describe('bulk actions and reducer effects', () => { it('selects only matching visible roots under search and clears selection', () => { const state = buildInitialState({ tree: buildPickerTree(pages()), - existingRootPageIds: [IDS.marketing], - currentCrawlMode: 'selected_roots', + existingSelectedIds: [IDS.marketing], }); const searching = { ...state, @@ -188,36 +188,35 @@ describe('bulk actions and reducer effects', () => { expect([...selectNone(selected).checked]).toEqual([]); }); - it('returns save immediately for selected_roots and requires confirmation for all_accessible', () => { - const selectedRoots = toggleChecked( + it('saves immediately when confirm is not required and prompts confirmation when requireConfirmOnSave is true', () => { + const noConfirm = toggleChecked( buildInitialState({ tree: buildPickerTree(pages()), - existingRootPageIds: [], - currentCrawlMode: 'selected_roots', + existingSelectedIds: [], }), IDS.marketing, 1000, ); - expect(reducer(selectedRoots, 'save-request')).toEqual({ - next: selectedRoots, + expect(reducer(noConfirm, 'save-request')).toEqual({ + next: noConfirm, effect: 'save', }); - const allAccessible = { - ...selectedRoots, - currentCrawlMode: 'all_accessible' as const, + const confirmRequired = { + ...noConfirm, + requireConfirmOnSave: true, }; - const confirm = reducer(allAccessible, 'save-request'); + const confirm = reducer(confirmRequired, 'save-request'); expect(confirm).toEqual({ - next: { ...allAccessible, pendingConfirm: 'mode-switch' }, + next: { ...confirmRequired, pendingConfirm: 'save-confirm' }, effect: null, }); expect(reducer(confirm.next, 'save-cancel')).toEqual({ - next: { ...allAccessible, pendingConfirm: null }, + next: { ...confirmRequired, pendingConfirm: null }, effect: null, }); expect(reducer(confirm.next, 'save-confirm')).toEqual({ - next: { ...allAccessible, pendingConfirm: null }, + next: { ...confirmRequired, pendingConfirm: null }, effect: 'save', }); }); @@ -225,8 +224,7 @@ describe('bulk actions and reducer effects', () => { it('prompts skip-empty confirmation on empty save, updates search state, and quits without saving', () => { const state = buildInitialState({ tree: buildPickerTree(pages()), - existingRootPageIds: [], - currentCrawlMode: 'selected_roots', + existingSelectedIds: [], }); const emptySave = reducer(state, 'save-request'); @@ -254,16 +252,33 @@ describe('bulk actions and reducer effects', () => { }); }); + it('treats skip-empty confirmation as a save with empty selection when skipEmptyAction is save-empty', () => { + const state = buildInitialState({ + tree: buildPickerTree(pages()), + existingSelectedIds: [], + skipEmptyAction: 'save-empty', + }); + + const emptySave = reducer(state, 'save-request'); + expect(emptySave).toEqual({ + next: { ...state, pendingConfirm: 'skip-empty' }, + effect: null, + }); + expect(reducer(emptySave.next, 'save-confirm')).toEqual({ + next: { ...state, pendingConfirm: null }, + effect: 'save', + }); + }); + it('clears transient hints only when their expiry time has passed', () => { const state = buildInitialState({ tree: buildPickerTree(pages()), - existingRootPageIds: [], - currentCrawlMode: 'selected_roots', + existingSelectedIds: [], }); const withHint = { ...state, transientHint: { - text: 'Select at least one page or press esc to cancel', + text: 'Select at least one item or press esc to cancel', expiresAt: 11500, }, }; diff --git a/packages/cli/src/notion-page-picker-tree.ts b/packages/cli/src/tree-picker-state.ts similarity index 86% rename from packages/cli/src/notion-page-picker-tree.ts rename to packages/cli/src/tree-picker-state.ts index 738ab723..9d9b3c68 100644 --- a/packages/cli/src/notion-page-picker-tree.ts +++ b/packages/cli/src/tree-picker-state.ts @@ -1,11 +1,11 @@ -export interface NotionPickerPageInput { +export interface TreePickerNodeInput { id: string; title?: string | null; archived?: boolean; parentId?: string | null; } -interface NotionPickerNode { +export interface TreePickerNode { id: string; title: string; archived: boolean; @@ -15,17 +15,22 @@ interface NotionPickerNode { path: string; } +type PendingConfirmKind = 'save-confirm' | 'skip-empty'; + +export type SkipEmptyAction = 'quit' | 'save-empty'; + export interface PickerState { - tree: NotionPickerNode[]; - byId: Map; + tree: TreePickerNode[]; + byId: Map; expanded: Set; checked: Set; cursorId: string; search: { editing: boolean; query: string }; - pendingConfirm: 'mode-switch' | 'skip-empty' | null; + pendingConfirm: PendingConfirmKind | null; preLoadWarnings: string[]; transientHint: { text: string; expiresAt: number } | null; - currentCrawlMode: 'all_accessible' | 'selected_roots'; + requireConfirmOnSave: boolean; + skipEmptyAction: SkipEmptyAction; } export type PickerCommand = @@ -65,25 +70,12 @@ const TRANSIENT_HINT_DURATION_MS = 2500; const collator = new Intl.Collator('en', { sensitivity: 'base', numeric: true }); -function normalizePageId(value: string): string { - const trimmed = value.trim(); - const compact = trimmed.replace(/-/g, ''); - if (/^[0-9a-fA-F]{32}$/.test(compact)) { - const lower = compact.toLowerCase(); - return `${lower.slice(0, 8)}-${lower.slice(8, 12)}-${lower.slice(12, 16)}-${lower.slice( - 16, - 20, - )}-${lower.slice(20)}`; - } - return trimmed; -} - function titleValue(value: string | null | undefined): string { const trimmed = value?.trim() ?? ''; return trimmed.length > 0 ? trimmed : 'Untitled'; } -function sortedNodeIds(ids: string[], nodes: Map): string[] { +function sortedNodeIds(ids: string[], nodes: Map): string[] { return [...ids].sort((leftId, rightId) => { const left = nodes.get(leftId); const right = nodes.get(rightId); @@ -107,7 +99,7 @@ export function clearExpiredTransientHint(state: PickerState, now = Date.now()): return cloneState(state, { transientHint: null }); } -function ancestorsOf(nodeId: string, byId: Map): string[] { +function ancestorsOf(nodeId: string, byId: Map): string[] { const ancestors: string[] = []; let parentId = byId.get(nodeId)?.parentId ?? null; const seen = new Set(); @@ -119,7 +111,7 @@ function ancestorsOf(nodeId: string, byId: Map): strin return ancestors; } -function descendantsOf(nodeId: string, byId: Map): string[] { +function descendantsOf(nodeId: string, byId: Map): string[] { const result: string[] = []; const stack = [...(byId.get(nodeId)?.childIds ?? [])].reverse(); while (stack.length > 0) { @@ -152,18 +144,18 @@ function matchingIds(state: PickerState): Set { ); } -export function buildPickerTree(searchResults: NotionPickerPageInput[]): NotionPickerNode[] { +export function buildPickerTree(inputs: TreePickerNodeInput[]): TreePickerNode[] { const nodes = new Map(); - for (const result of searchResults) { - const id = normalizePageId(result.id); - if (nodes.has(id)) { + for (const result of inputs) { + const id = result.id.trim(); + if (id.length === 0 || nodes.has(id)) { continue; } nodes.set(id, { id, title: titleValue(result.title), archived: result.archived === true, - parentId: result.parentId ? normalizePageId(result.parentId) : null, + parentId: result.parentId ? result.parentId.trim() : null, childIds: [], }); } @@ -202,7 +194,7 @@ export function buildPickerTree(searchResults: NotionPickerPageInput[]): NotionP [...nodes.values()].filter((node) => node.parentId === null).map((node) => node.id), nodes, ); - const tree: NotionPickerNode[] = []; + const tree: TreePickerNode[] = []; function visit(nodeId: string, depth: number, pathPrefix: string[]): void { const raw = nodes.get(nodeId); @@ -210,7 +202,7 @@ export function buildPickerTree(searchResults: NotionPickerPageInput[]): NotionP return; } const path = [...pathPrefix, raw.title].join(' / '); - const node: NotionPickerNode = { + const node: TreePickerNode = { id: raw.id, title: raw.title, archived: raw.archived, @@ -232,11 +224,11 @@ export function buildPickerTree(searchResults: NotionPickerPageInput[]): NotionP return tree; } -export function isAncestorChecked(nodeId: string, checked: Set, byId: Map): boolean { +export function isAncestorChecked(nodeId: string, checked: Set, byId: Map): boolean { return ancestorsOf(nodeId, byId).some((ancestorId) => checked.has(ancestorId)); } -function checkedAncestor(nodeId: string, state: PickerState): NotionPickerNode | null { +function checkedAncestor(nodeId: string, state: PickerState): TreePickerNode | null { for (const ancestorId of ancestorsOf(nodeId, state.byId)) { if (state.checked.has(ancestorId)) { return state.byId.get(ancestorId) ?? null; @@ -247,7 +239,7 @@ function checkedAncestor(nodeId: string, state: PickerState): NotionPickerNode | export function canToggle(nodeId: string, state: PickerState): { ok: true } | { ok: false; reason: string } { if (!state.byId.has(nodeId)) { - return { ok: false, reason: 'Page not found' }; + return { ok: false, reason: 'Node not found' }; } const ancestor = checkedAncestor(nodeId, state); if (ancestor) { @@ -276,7 +268,7 @@ export function toggleChecked(state: PickerState, nodeId: string, now = Date.now return cloneState(state, { checked, transientHint: null }); } -export function flattenSelection(checked: Set, byId: Map): string[] { +export function flattenSelection(checked: Set, byId: Map): string[] { const result: string[] = []; for (const node of byId.values()) { if (checked.has(node.id) && !isAncestorChecked(node.id, checked, byId)) { @@ -402,16 +394,21 @@ export function moveCursor(state: PickerState, dir: 'up' | 'down' | 'left' | 'ri } export function buildInitialState(args: { - tree: NotionPickerNode[]; - existingRootPageIds: string[]; - currentCrawlMode?: 'all_accessible' | 'selected_roots'; + tree: TreePickerNode[]; + existingSelectedIds: string[]; + requireConfirmOnSave?: boolean; + skipEmptyAction?: SkipEmptyAction; + staleWarning?: (staleCount: number) => string; }): PickerState { const byId = new Map(args.tree.map((node) => [node.id, node])); const checked = new Set(); let staleCount = 0; - for (const rawId of args.existingRootPageIds) { - const id = normalizePageId(rawId); + for (const rawId of args.existingSelectedIds) { + const id = rawId.trim(); + if (id.length === 0) { + continue; + } if (byId.has(id)) { checked.add(id); } else { @@ -427,6 +424,12 @@ export function buildInitialState(args: { } } + const preLoadWarnings: string[] = []; + if (staleCount > 0) { + const warning = args.staleWarning ? args.staleWarning(staleCount) : `${staleCount} stored selections no longer visible`; + preLoadWarnings.push(warning); + } + return { tree: args.tree, byId, @@ -435,16 +438,18 @@ export function buildInitialState(args: { cursorId: args.tree[0]?.id ?? '', search: { editing: false, query: '' }, pendingConfirm: null, - preLoadWarnings: staleCount > 0 ? [`${staleCount} stored root_page_ids no longer visible`] : [], + preLoadWarnings, transientHint: null, - currentCrawlMode: args.currentCrawlMode ?? 'selected_roots', + requireConfirmOnSave: args.requireConfirmOnSave ?? false, + skipEmptyAction: args.skipEmptyAction ?? 'quit', }; } export function reducer(state: PickerState, cmd: PickerCommand, now = Date.now()): { next: PickerState; effect: PickerEffect } { if (state.pendingConfirm) { if (cmd === 'save-confirm') { - const effect: PickerEffect = state.pendingConfirm === 'skip-empty' ? 'quit-without-save' : 'save'; + const effect: PickerEffect = + state.pendingConfirm === 'skip-empty' ? (state.skipEmptyAction === 'save-empty' ? 'save' : 'quit-without-save') : 'save'; return { next: cloneState(state, { pendingConfirm: null }), effect }; } if (cmd === 'save-cancel') { @@ -501,8 +506,8 @@ export function reducer(state: PickerState, cmd: PickerCommand, now = Date.now() if (state.checked.size === 0) { return { next: cloneState(state, { pendingConfirm: 'skip-empty' }), effect: null }; } - if (state.currentCrawlMode === 'all_accessible') { - return { next: cloneState(state, { pendingConfirm: 'mode-switch' }), effect: null }; + if (state.requireConfirmOnSave) { + return { next: cloneState(state, { pendingConfirm: 'save-confirm' }), effect: null }; } return { next: state, effect: 'save' }; case 'save-confirm': diff --git a/packages/cli/src/tree-picker-tui.test.tsx b/packages/cli/src/tree-picker-tui.test.tsx new file mode 100644 index 00000000..8c4f8d1e --- /dev/null +++ b/packages/cli/src/tree-picker-tui.test.tsx @@ -0,0 +1,361 @@ +/* @jsxImportSource react */ +import { render as renderInkTest } from 'ink-testing-library'; +import { type ReactNode } from 'react'; +import { describe, expect, it, vi } from 'vitest'; +import { buildInitialState, buildPickerTree, type TreePickerNodeInput } from './tree-picker-state.js'; +import { + TreePickerApp, + renderTreePickerTui, + resolveTreePickerWidth, + sanitizeTreePickerTuiError, + treePickerCommandForInkInput, + windowItems, + windowOffset, + type TreePickerChrome, + type TreePickerInkInstance, + type TreePickerInkRenderOptions, +} from './tree-picker-tui.js'; + +const IDS = { + engineering: '11111111-1111-1111-1111-111111111111', + architecture: '22222222-2222-2222-2222-222222222222', + marketing: '33333333-3333-3333-3333-333333333333', + finance: '44444444-4444-4444-4444-444444444444', + ops: '55555555-5555-5555-5555-555555555555', + sales: '66666666-6666-6666-6666-666666666666', + support: '77777777-7777-7777-7777-777777777777', + product: '88888888-8888-8888-8888-888888888888', + design: '99999999-9999-9999-9999-999999999999', +}; + +function pages(): TreePickerNodeInput[] { + return [ + { id: IDS.engineering, title: 'Engineering Docs', archived: false, parentId: null }, + { id: IDS.architecture, title: 'Architecture', archived: false, parentId: IDS.engineering }, + { id: IDS.marketing, title: 'Marketing', archived: false, parentId: null }, + ]; +} + +function manyPages(): TreePickerNodeInput[] { + return [ + { id: IDS.engineering, title: 'Engineering Docs', archived: false, parentId: null }, + { id: IDS.architecture, title: 'Architecture', archived: false, parentId: IDS.engineering }, + { id: IDS.marketing, title: 'Marketing', archived: false, parentId: null }, + { id: IDS.finance, title: 'Finance', archived: false, parentId: null }, + { id: IDS.ops, title: 'Operations', archived: false, parentId: null }, + { id: IDS.sales, title: 'Sales', archived: false, parentId: null }, + { id: IDS.support, title: 'Support', archived: false, parentId: null }, + { id: IDS.product, title: 'Product', archived: false, parentId: null }, + { id: IDS.design, title: 'Design', archived: false, parentId: null }, + ]; +} + +function state(options: { requireConfirmOnSave?: boolean } = {}) { + return buildInitialState({ + tree: buildPickerTree(pages()), + existingSelectedIds: [], + requireConfirmOnSave: options.requireConfirmOnSave ?? false, + }); +} + +function chrome(overrides: Partial = {}): TreePickerChrome { + return { + title: 'Select items', + subtitleLines: ['Source: Test'], + ...overrides, + }; +} + +async function waitForInkInput(): Promise { + await new Promise((resolve) => setTimeout(resolve, 10)); +} + +function fakeInkInstance(): TreePickerInkInstance { + return { + rerender: vi.fn(), + unmount: vi.fn(), + waitUntilExit: vi.fn(async () => undefined), + }; +} + +function normalizeFrameWrap(frame: string | undefined): string { + return frame?.replace(/\n/g, ' ').replace(/│ /g, '').replace(/ +/g, ' ') ?? ''; +} + +describe('treePickerCommandForInkInput', () => { + it('maps browse, search, and confirm input to reducer commands', () => { + expect(treePickerCommandForInkInput('', { downArrow: true }, state().search, null)).toBe('cursor-down'); + expect(treePickerCommandForInkInput('', { upArrow: true }, state().search, null)).toBe('cursor-up'); + expect(treePickerCommandForInkInput('', { rightArrow: true }, state().search, null)).toBe('cursor-right'); + expect(treePickerCommandForInkInput('', { leftArrow: true }, state().search, null)).toBe('cursor-left'); + expect(treePickerCommandForInkInput(' ', {}, state().search, null)).toBe('toggle-check'); + expect(treePickerCommandForInkInput('/', {}, state().search, null)).toBe('search-start'); + expect(treePickerCommandForInkInput('a', {}, state().search, null)).toBe('select-all-visible'); + expect(treePickerCommandForInkInput('n', {}, state().search, null)).toBe('select-none'); + expect(treePickerCommandForInkInput('', { return: true }, state().search, null)).toBe('save-request'); + expect(treePickerCommandForInkInput('', { escape: true }, state().search, null)).toBe('quit'); + expect(treePickerCommandForInkInput('c', { ctrl: true }, state().search, null)).toBe('quit'); + expect(treePickerCommandForInkInput('s', {}, state().search, null)).toBeNull(); + expect(treePickerCommandForInkInput('q', {}, state().search, null)).toBeNull(); + + expect(treePickerCommandForInkInput('x', {}, { editing: true, query: '' }, null)).toEqual({ + type: 'search-input', + value: 'x', + }); + expect(treePickerCommandForInkInput('', { backspace: true }, { editing: true, query: 'x' }, null)).toBe( + 'search-backspace', + ); + expect(treePickerCommandForInkInput('', { return: true }, { editing: true, query: 'x' }, null)).toBe( + 'search-submit', + ); + expect(treePickerCommandForInkInput('', { escape: true }, { editing: true, query: 'x' }, null)).toBe( + 'search-cancel', + ); + + expect(treePickerCommandForInkInput('y', {}, state().search, 'save-confirm')).toBe('save-confirm'); + expect(treePickerCommandForInkInput('', { return: true }, state().search, 'save-confirm')).toBe('save-confirm'); + expect(treePickerCommandForInkInput('n', {}, state().search, 'save-confirm')).toBe('save-cancel'); + }); +}); + +describe('window helpers', () => { + it('centers the selected row and returns the visible slice', () => { + expect(windowOffset(20, 10, 5)).toBe(8); + expect(windowItems(['a', 'b', 'c', 'd', 'e'], 3, 3)).toEqual({ items: ['c', 'd', 'e'], offset: 2 }); + }); + + it('clamps picker width to the design rule', () => { + expect(resolveTreePickerWidth(200)).toBe(120); + expect(resolveTreePickerWidth(100)).toBe(96); + expect(resolveTreePickerWidth(50)).toBe(60); + expect(resolveTreePickerWidth(undefined)).toBe(96); + }); +}); + +describe('TreePickerApp', () => { + it('renders chrome title, subtitle, warnings, help, and row glyphs', () => { + const initialState = { + ...state(), + preLoadWarnings: ['1 stale stored selections - they will be removed if you save'], + }; + const { lastFrame } = renderInkTest( + , + ); + + const frame = lastFrame() ?? ''; + expect(frame).toContain('Select fancy widgets'); + expect(frame).toContain('Workspace: Design Workspace'); + expect(frame).toContain('5000-page cap reached - some pages not shown'); + expect(frame).toContain('1 stale stored selections - they will be removed if you save'); + expect(frame).toContain('◻ Engineering Docs ▸ (1)'); + expect(frame).toContain('◻ Marketing'); + expect(normalizeFrameWrap(frame)).toContain( + 'Right Arrow to expand, Up/Down to move, Space to select or unselect, Slash to filter, Enter to confirm, Escape to go back, or Ctrl+C to exit.', + ); + }); + + it('renders custom help text when supplied', () => { + const { lastFrame } = renderInkTest( + , + ); + expect(lastFrame() ?? '').toContain('Bespoke instructions here.'); + }); + + it('renders checked parents and locked descendants with locked glyphs', () => { + const initialState = { + ...state(), + checked: new Set([IDS.engineering]), + expanded: new Set([IDS.engineering]), + }; + const { lastFrame } = renderInkTest( + , + ); + + const frame = lastFrame() ?? ''; + expect(frame).toContain('◼ Engineering Docs ▾'); + expect(frame).toContain(' ◼ Architecture'); + }); + + it('supports keyboard selection, confirm-on-save, and save callback', async () => { + const onExit = vi.fn(); + const { stdin, lastFrame } = renderInkTest( + + `Confirm: ${current.checked.size} item${current.checked.size === 1 ? '' : 's'}? Press Enter or Escape.`, + })} + terminalRows={24} + terminalWidth={100} + onExit={onExit} + />, + ); + + stdin.write(' '); + await waitForInkInput(); + expect(lastFrame()).toContain('◼ Engineering Docs'); + + stdin.write('\r'); + await waitForInkInput(); + expect(normalizeFrameWrap(lastFrame())).toContain('Confirm: 1 item? Press Enter or Escape.'); + + stdin.write('y'); + await waitForInkInput(); + expect(onExit).toHaveBeenCalledWith({ kind: 'save', selectedIds: [IDS.engineering] }); + }); + + it('uses the chrome-supplied skip-empty message and quits on confirm', async () => { + const onExit = vi.fn(); + const { stdin, lastFrame } = renderInkTest( + , + ); + + stdin.write('\r'); + await waitForInkInput(); + expect(normalizeFrameWrap(lastFrame())).toContain('No selections. Skip or back?'); + expect(onExit).not.toHaveBeenCalled(); + + stdin.write('n'); + await waitForInkInput(); + expect(lastFrame()).not.toContain('No selections. Skip or back?'); + expect(onExit).not.toHaveBeenCalled(); + + stdin.write('\r'); + await waitForInkInput(); + expect(lastFrame()).toContain('No selections. Skip or back?'); + + stdin.write('\r'); + await waitForInkInput(); + expect(onExit).toHaveBeenCalledWith({ kind: 'quit' }); + }); + + it('renders row-window overflow indicators when the visible list is clipped', async () => { + const onExit = vi.fn(); + const initialState = buildInitialState({ + tree: buildPickerTree(manyPages()), + existingSelectedIds: [], + }); + initialState.expanded = new Set([IDS.engineering]); + const { stdin, lastFrame } = renderInkTest( + , + ); + + expect(lastFrame()).toContain('↓ 4 more'); + + stdin.write(''); + stdin.write(''); + stdin.write(''); + stdin.write(''); + await waitForInkInput(); + + const frame = lastFrame() ?? ''; + expect(frame).toContain('↑ '); + expect(frame).toContain('↓ '); + expect(onExit).not.toHaveBeenCalled(); + }); + + it('quits without saving on Ctrl+C', async () => { + const onExit = vi.fn(); + const { stdin } = renderInkTest( + , + ); + + stdin.write(''); + await waitForInkInput(); + expect(onExit).toHaveBeenCalledWith({ kind: 'quit' }); + }); +}); + +describe('renderTreePickerTui', () => { + it('returns the app result from the Ink runtime', async () => { + const io = { + stdin: { isTTY: true, setRawMode: vi.fn() }, + stdout: { isTTY: true, columns: 100, rows: 24, write: vi.fn() }, + stderr: { write: vi.fn() }, + }; + const renderInk = vi.fn((_tree: ReactNode, _options: TreePickerInkRenderOptions) => fakeInkInstance()); + + await expect( + renderTreePickerTui( + { initialState: state(), chrome: chrome() }, + io, + { renderInk }, + ), + ).resolves.toEqual({ kind: 'quit' }); + expect(renderInk).toHaveBeenCalledOnce(); + }); + + it('sanitizes render errors and uses the supplied scripted-mode hint', async () => { + expect(sanitizeTreePickerTuiError(new Error('token=secret https://api.example.com/v1/search'))).toBe( + '[redacted] [redacted-url]', + ); + }); + + it('falls back to quit with the scripted-mode hint when Ink cannot initialize', async () => { + let stderr = ''; + const io = { + stdin: { isTTY: false, setRawMode: vi.fn() }, + stdout: { isTTY: false, columns: 100, rows: 24, write: vi.fn() }, + stderr: { + write(chunk: string) { + stderr += chunk; + }, + }, + }; + + await expect( + renderTreePickerTui( + { initialState: state(), chrome: chrome() }, + io, + { + renderInk: vi.fn(() => { + throw new Error('token=secret'); + }), + scriptedModeHint: 'Use --no-input --foo bar for scripted mode.', + }, + ), + ).resolves.toEqual({ kind: 'quit' }); + expect(stderr).toContain('Use --no-input --foo bar for scripted mode.'); + expect(stderr).not.toContain('secret'); + }); +}); diff --git a/packages/cli/src/notion-page-picker-tui.tsx b/packages/cli/src/tree-picker-tui.tsx similarity index 67% rename from packages/cli/src/notion-page-picker-tui.tsx rename to packages/cli/src/tree-picker-tui.tsx index d627d200..9cdbef8d 100644 --- a/packages/cli/src/notion-page-picker-tui.tsx +++ b/packages/cli/src/tree-picker-tui.tsx @@ -9,7 +9,7 @@ import { visibleNodeIds, type PickerCommand, type PickerState, -} from './notion-page-picker-tree.js'; +} from './tree-picker-state.js'; import type { KtxCliIo } from './cli-runtime.js'; const COLOR_THEME = { @@ -28,9 +28,15 @@ const NO_COLOR_THEME = { warning: 'white', } as const; -type NotionPickerTheme = Record; +type TreePickerTheme = Record; -export interface NotionPickerTuiIo extends KtxCliIo { +const DEFAULT_TREE_PICKER_HELP_TEXT = + 'Right Arrow to expand, Up/Down to move, Space to select or unselect, Slash to filter, Enter to confirm, Escape to go back, or Ctrl+C to exit.'; + +const DEFAULT_SKIP_EMPTY_MESSAGE = + 'Nothing selected. Skip this step? Press Enter to skip or Escape to go back.'; + +export interface TreePickerTuiIo extends KtxCliIo { stdin?: { isTTY?: boolean; setRawMode?(value: boolean): void }; stdout: KtxCliIo['stdout'] & { isTTY?: boolean; columns?: number; rows?: number }; } @@ -47,58 +53,54 @@ interface InkKey { delete?: boolean; } -export type PickerRenderResult = { kind: 'save'; rootPageIds: string[] } | { kind: 'quit' }; +export type TreePickerResult = { kind: 'save'; selectedIds: string[] } | { kind: 'quit' }; -export interface PickerRenderInput { - initialState: PickerState; - connectionId: string; - workspaceLabel: string; - cappedAtCount: number | null; - currentCrawlMode: 'all_accessible' | 'selected_roots'; +export interface TreePickerChrome { + title: string; + helpText?: string; + subtitleLines?: readonly string[]; + warningLines?: readonly string[]; + confirmSaveMessage?: (state: PickerState) => string; + skipEmptyMessage?: string; } -interface NotionPickerAppProps extends PickerRenderInput { +export interface TreePickerRenderInput { + initialState: PickerState; + chrome: TreePickerChrome; +} + +interface TreePickerAppProps extends TreePickerRenderInput { terminalRows?: number; terminalWidth?: number; env?: NodeJS.ProcessEnv; - onExit(result: PickerRenderResult): void; + onExit(result: TreePickerResult): void; } -export interface NotionPickerInkInstance { +export interface TreePickerInkInstance { rerender(tree: ReactNode): void; unmount(): void; waitUntilExit(): Promise; } -export interface NotionPickerInkRenderOptions { - stdin?: NotionPickerTuiIo['stdin']; - stdout: NotionPickerTuiIo['stdout']; - stderr: NotionPickerTuiIo['stderr']; +export interface TreePickerInkRenderOptions { + stdin?: TreePickerTuiIo['stdin']; + stdout: TreePickerTuiIo['stdout']; + stderr: TreePickerTuiIo['stderr']; exitOnCtrlC: boolean; patchConsole: boolean; maxFps: number; alternateScreen: boolean; } -function resolveTheme(env: NodeJS.ProcessEnv = process.env): NotionPickerTheme { +function resolveTheme(env: NodeJS.ProcessEnv = process.env): TreePickerTheme { return env.NO_COLOR || env.TERM === 'dumb' ? NO_COLOR_THEME : COLOR_THEME; } -export function resolveNotionPickerWidth(columns: number | undefined): number { +export function resolveTreePickerWidth(columns: number | undefined): number { const resolvedColumns = columns ?? 100; return Math.max(60, Math.min(120, resolvedColumns - 4)); } -function staleWarningText(warning: string): string { - return warning.includes('stored root_page_ids no longer visible') - ? `${warning} - they will be removed if you save` - : warning; -} - -function selectedPageCountText(count: number): string { - return `${count} selected ${count === 1 ? 'page' : 'pages'}`; -} - function rowMatchesSearch(state: PickerState, nodeId: string): boolean { const query = state.search.query.trim().toLocaleLowerCase(); if (!query) { @@ -111,7 +113,7 @@ function rowMatchesSearch(state: PickerState, nodeId: string): boolean { return node.title.toLocaleLowerCase().includes(query) || node.path.toLocaleLowerCase().includes(query); } -export function sanitizeNotionPickerTuiError(error: unknown): string { +export function sanitizeTreePickerTuiError(error: unknown): string { const message = error instanceof Error ? error.message : String(error); return message .replace(/[a-z][a-z0-9+.-]*:\/\/[^\s]+/gi, '[redacted-url]') @@ -134,7 +136,7 @@ function truncateText(value: string, width: number): string { return `${value.slice(0, width - 3)}...`; } -export function notionPickerCommandForInkInput( +export function treePickerCommandForInkInput( input: string, key: InkKey, search: PickerState['search'], @@ -152,7 +154,7 @@ export function notionPickerCommandForInkInput( if (key.backspace || key.delete) return 'search-backspace'; if (key.downArrow) return 'cursor-down'; if (key.upArrow) return 'cursor-up'; - if (input.length === 1 && input >= ' ' && input !== '\u007f') return { type: 'search-input', value: input }; + if (input.length === 1 && input >= ' ' && input !== '') return { type: 'search-input', value: input }; return null; } if (key.ctrl === true && input === 'c') return 'quit'; @@ -169,7 +171,7 @@ export function notionPickerCommandForInkInput( return null; } -function PickerRow(props: { state: PickerState; nodeId: string; width: number; theme: NotionPickerTheme }): ReactNode { +function PickerRow(props: { state: PickerState; nodeId: string; width: number; theme: TreePickerTheme }): ReactNode { const node = props.state.byId.get(props.nodeId); if (!node) return null; const focused = props.state.cursorId === node.id; @@ -177,14 +179,14 @@ function PickerRow(props: { state: PickerState; nodeId: string; width: number; t const checked = props.state.checked.has(node.id); const isSelected = checked || locked; const glyph = isSelected ? '◼' : '◻'; - const glyphColor = locked ? props.theme.muted : checked ? props.theme.selected : props.theme.muted; + const glyphColor = checked || locked ? props.theme.selected : props.theme.muted; const childAffordance = node.childIds.length > 0 ? (props.state.expanded.has(node.id) ? ' ▾' : ` ▸ (${node.childIds.length})`) : ''; const indent = ' '.repeat(node.depth * 2); - const titleColor = focused ? props.theme.text : props.theme.muted; + const titleColor = focused ? props.theme.active : props.theme.text; const inverse = rowMatchesSearch(props.state, node.id); - const prefixWidth = indent.length + 2; - const title = truncateText(`${node.title}${childAffordance}`, Math.max(10, props.width - prefixWidth)); + const prefixWidth = indent.length + 2 + childAffordance.length; + const title = truncateText(node.title, Math.max(10, props.width - prefixWidth)); return ( @@ -192,30 +194,32 @@ function PickerRow(props: { state: PickerState; nodeId: string; width: number; t {indent} {glyph} - + {' '} {title} + {childAffordance.length > 0 ? {childAffordance} : null} ); } -export function NotionPickerApp(props: NotionPickerAppProps): ReactNode { +export function TreePickerApp(props: TreePickerAppProps): ReactNode { const app = useApp(); const [state, setState] = useState(props.initialState); const stateRef = useRef(state); const theme = useMemo(() => resolveTheme(props.env), [props.env]); const visibleIds = visibleNodeIds(state); const selectedIndex = Math.max(0, visibleIds.indexOf(state.cursorId)); - const reservedRows = state.pendingConfirm === 'mode-switch' ? 9 : 8; + const reservedRows = state.pendingConfirm === 'save-confirm' ? 10 : 9; const visibleRows = Math.max(5, Math.min(12, (props.terminalRows ?? 24) - reservedRows)); const rows = windowItems(visibleIds, selectedIndex, visibleRows); const hiddenAbove = rows.offset; const hiddenBelow = Math.max(0, visibleIds.length - rows.offset - rows.items.length); const searchMatchCount = filterTree(state).visibleIds.size; - const width = resolveNotionPickerWidth(props.terminalWidth); + const width = resolveTreePickerWidth(props.terminalWidth); const showSearch = state.search.editing || state.search.query.trim().length > 0; - const selectedCount = flattenSelection(state.checked, state.byId).length; + const helpText = props.chrome.helpText ?? DEFAULT_TREE_PICKER_HELP_TEXT; + const skipEmptyMessage = props.chrome.skipEmptyMessage ?? DEFAULT_SKIP_EMPTY_MESSAGE; stateRef.current = state; @@ -244,7 +248,7 @@ export function NotionPickerApp(props: NotionPickerAppProps): ReactNode { }, [state.transientHint?.expiresAt]); useInput((input, key) => { - const command = notionPickerCommandForInkInput(input, key, stateRef.current.search, stateRef.current.pendingConfirm); + const command = treePickerCommandForInkInput(input, key, stateRef.current.search, stateRef.current.pendingConfirm); if (!command) { return; } @@ -252,7 +256,7 @@ export function NotionPickerApp(props: NotionPickerAppProps): ReactNode { stateRef.current = next; setState(next); if (effect === 'save') { - props.onExit({ kind: 'save', rootPageIds: flattenSelection(next.checked, next.byId) }); + props.onExit({ kind: 'save', selectedIds: flattenSelection(next.checked, next.byId) }); app.exit(); return; } @@ -266,7 +270,7 @@ export function NotionPickerApp(props: NotionPickerAppProps): ReactNode { ◆ - Select Notion pages to ingest + {props.chrome.title} - - Right Arrow to expand, Up/Down to move, Space to select or unselect, Slash to filter, Enter to confirm, Escape - to go back, or Ctrl+C to exit. - + {helpText} - Workspace: {props.workspaceLabel} - {props.cappedAtCount ? ( - {props.cappedAtCount}-page cap reached - some pages not shown - ) : null} + {(props.chrome.subtitleLines ?? []).map((line, idx) => ( + + {line} + + ))} + {(props.chrome.warningLines ?? []).map((line, idx) => ( + + {line} + + ))} {state.preLoadWarnings.map((warning) => ( - {staleWarningText(warning)} + {warning} ))} {showSearch ? ( @@ -301,20 +308,20 @@ export function NotionPickerApp(props: NotionPickerAppProps): ReactNode { ({searchMatchCount} matches) ) : null} + {hiddenAbove > 0 ? ↑ {hiddenAbove} more : null} {rows.items.map((nodeId) => ( ))} {hiddenBelow > 0 ? ↓ {hiddenBelow} more : null} - {state.pendingConfirm === 'mode-switch' ? ( + {state.pendingConfirm === 'save-confirm' ? ( - Switch crawl_mode from all_accessible to selected_roots? Will limit ingest to{' '} - {selectedPageCountText(selectedCount)}. Press Enter to confirm or Escape to go back. + {props.chrome.confirmSaveMessage + ? props.chrome.confirmSaveMessage(state) + : 'Confirm save? Press Enter to confirm or Escape to go back.'} ) : null} - {state.pendingConfirm === 'skip-empty' ? ( - Nothing selected. Skip this step? Press Enter to skip or Escape to go back. - ) : null} + {state.pendingConfirm === 'skip-empty' ? {skipEmptyMessage} : null} {state.transientHint ? {state.transientHint.text} : null} └ @@ -322,7 +329,7 @@ export function NotionPickerApp(props: NotionPickerAppProps): ReactNode { ); } -function renderInk(tree: ReactNode, options: NotionPickerInkRenderOptions): NotionPickerInkInstance { +function renderInk(tree: ReactNode, options: TreePickerInkRenderOptions): TreePickerInkInstance { return renderInkRuntime(tree, { stdin: options.stdin as NodeJS.ReadStream | undefined, stdout: options.stdout as NodeJS.WriteStream, @@ -331,19 +338,24 @@ function renderInk(tree: ReactNode, options: NotionPickerInkRenderOptions): Noti patchConsole: options.patchConsole, maxFps: options.maxFps, alternateScreen: options.alternateScreen, - }) as NotionPickerInkInstance; + }) as TreePickerInkInstance; } -export async function renderNotionPickerTui( - input: PickerRenderInput, - io: NotionPickerTuiIo, - options: { renderInk?: (tree: ReactNode, options: NotionPickerInkRenderOptions) => NotionPickerInkInstance } = {}, -): Promise { - let result: PickerRenderResult = { kind: 'quit' }; - let instance: NotionPickerInkInstance | null = null; +export interface RenderTreePickerOptions { + renderInk?: (tree: ReactNode, options: TreePickerInkRenderOptions) => TreePickerInkInstance; + scriptedModeHint?: string; +} + +export async function renderTreePickerTui( + input: TreePickerRenderInput, + io: TreePickerTuiIo, + options: RenderTreePickerOptions = {}, +): Promise { + let result: TreePickerResult = { kind: 'quit' }; + let instance: TreePickerInkInstance | null = null; try { instance = (options.renderInk ?? renderInk)( - for scripted mode. ${sanitizeNotionPickerTuiError(error)}\n`, - ); + const hint = options.scriptedModeHint ?? 'Picker requires a TTY.'; + io.stderr.write(`${hint} ${sanitizeTreePickerTuiError(error)}\n`); return { kind: 'quit' }; } } From ed690ef60c54e4a9778034894f212c8911d94899 Mon Sep 17 00:00:00 2001 From: Luca Martial <48870843+luca-martial@users.noreply.github.com> Date: Wed, 13 May 2026 18:47:58 -0400 Subject: [PATCH 7/7] feat(cli): redesign ktx status output UX (#80) * feat(cli): redesign ktx status output with grouped checks and color Replace flat PASS/FAIL/WARN text output with a grouped, symbol-based layout (Environment, Project, Semantic search, Query history). Passing groups collapse to a single summary line; failing groups expand to show individual checks with fix hints. Adds --verbose flag to show all checks including passing ones, color support for TTY terminals, a dedicated setup-mode report that guides users toward `ktx setup`, and timing info. Co-Authored-By: Claude Opus 4.6 (1M context) * refactor(cli): extract project checks and historic SQL doctor into status-project Move project-level doctor checks, semantic search embedding checks, and historic SQL doctor logic from doctor.ts into a dedicated status-project.ts module. Removes historic-sql-doctor.ts and its test file, consolidating everything into the new module with its own tests. Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Claude Opus 4.6 (1M context) --- packages/cli/src/commands/status-commands.ts | 5 +- packages/cli/src/doctor.test.ts | 365 +++++------ packages/cli/src/doctor.ts | 433 ++++++++----- packages/cli/src/historic-sql-doctor.test.ts | 202 ------ packages/cli/src/historic-sql-doctor.ts | 167 ----- packages/cli/src/index.test.ts | 4 +- packages/cli/src/status-project.ts | 614 +++++++++++++++++++ 7 files changed, 1040 insertions(+), 750 deletions(-) delete mode 100644 packages/cli/src/historic-sql-doctor.test.ts delete mode 100644 packages/cli/src/historic-sql-doctor.ts create mode 100644 packages/cli/src/status-project.ts diff --git a/packages/cli/src/commands/status-commands.ts b/packages/cli/src/commands/status-commands.ts index d834e15b..52032e59 100644 --- a/packages/cli/src/commands/status-commands.ts +++ b/packages/cli/src/commands/status-commands.ts @@ -16,8 +16,9 @@ export function registerStatusCommands(program: Command, context: KtxCliCommandC .command('status') .description('Check current KTX setup and project readiness') .option('--json', 'Print JSON output', false) + .option('-v, --verbose', 'Show every check, including passing ones', false) .option('--no-input', 'Disable interactive terminal input') - .action(async (options: { json?: boolean; input?: boolean }, command) => { + .action(async (options: { json?: boolean; verbose?: boolean; input?: boolean }, command) => { const runner = context.deps.doctor ?? (await import('../doctor.js')).runKtxDoctor; const explicitOrEnvProjectDir = resolveCommandProjectDirOverride(command); const nearestProjectDir = explicitOrEnvProjectDir ? undefined : findNearestKtxProjectDir(process.cwd()); @@ -27,6 +28,7 @@ export function registerStatusCommands(program: Command, context: KtxCliCommandC { command: 'setup', outputMode: outputMode(options), + verbose: options.verbose === true, ...inputMode(options), }, context.io, @@ -40,6 +42,7 @@ export function registerStatusCommands(program: Command, context: KtxCliCommandC command: 'project', projectDir: resolveCommandProjectDir(command), outputMode: outputMode(options), + verbose: options.verbose === true, ...inputMode(options), }, context.io, diff --git a/packages/cli/src/doctor.test.ts b/packages/cli/src/doctor.test.ts index cd96e6d2..d5db8ab6 100644 --- a/packages/cli/src/doctor.test.ts +++ b/packages/cli/src/doctor.test.ts @@ -1,8 +1,7 @@ import { mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import type { KtxEmbeddingConfig, KtxEmbeddingHealthCheckOptions, KtxEmbeddingHealthCheckResult } from '@ktx/llm'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { formatDoctorReport, runKtxDoctor, @@ -31,53 +30,65 @@ function makeIo() { }; } -type EmbeddingHealthCheck = ( - config: KtxEmbeddingConfig, - options?: KtxEmbeddingHealthCheckOptions, -) => Promise; - -async function writeProjectConfig(projectDir: string, embeddingLines: string[]): Promise { - await writeFile( - join(projectDir, 'ktx.yaml'), - [ - 'project: warehouse', - 'connections:', - ' warehouse:', - ' driver: sqlite', - ' path: ./warehouse.db', - 'ingest:', - ' adapters:', - ' - live-database', - ' embeddings:', - ...embeddingLines.map((line) => ` ${line}`), - '', - ].join('\n'), - 'utf-8', - ); -} - describe('formatDoctorReport', () => { - it('prints exact fixes for failing setup checks', () => { + it('shows the failing check and its fix in plain output', () => { const checks: DoctorCheck[] = [ - { id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' }, + { id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127', group: 'toolchain' }, { id: 'native-sqlite', label: 'Native SQLite', status: 'fail', detail: 'Cannot load better-sqlite3', fix: 'Run: pnpm run native:rebuild', + group: 'toolchain', }, ]; - expect(formatDoctorReport({ title: 'KTX setup doctor', checks })).toBe( - [ - 'KTX setup doctor', - 'PASS Node 22+: v22.16.0 ABI 127', - 'FAIL Native SQLite: Cannot load better-sqlite3', - ' Fix: Run: pnpm run native:rebuild', - '', - ].join('\n'), - ); + const output = formatDoctorReport({ title: 'KTX status', checks }); + expect(output).toContain('KTX status'); + expect(output).toContain('✗ Environment'); + expect(output).toContain('1 of 2 need attention'); + expect(output).toContain('✗ Native SQLite: Cannot load better-sqlite3'); + expect(output).toContain('→ Run: pnpm run native:rebuild'); + expect(output).toContain('1 issue to fix.'); + }); + + it('lists what was checked when a group has all passing checks', () => { + const checks: DoctorCheck[] = [ + { id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0', group: 'toolchain' }, + { id: 'pnpm', label: 'pnpm 10.20+', status: 'pass', detail: '10.28.0', group: 'toolchain' }, + ]; + + const output = formatDoctorReport({ title: 'KTX status', checks }); + expect(output).toContain('✓ Environment'); + expect(output).toContain('Node 22+ · pnpm 10.20+'); + expect(output).not.toContain('v22.16.0'); + expect(output).toContain('Everything ready.'); + }); + + it('shows the underlying detail for a single-check group on the group line', () => { + const checks: DoctorCheck[] = [ + { + id: 'semantic-search-embeddings', + label: 'Semantic search embeddings', + status: 'pass', + detail: 'openai/text-embedding-3-small (1536d) probe succeeded', + group: 'search', + }, + ]; + + const output = formatDoctorReport({ title: 'KTX status', checks }); + expect(output).toContain('✓ Semantic search'); + expect(output).toContain('openai/text-embedding-3-small (1536d) probe succeeded'); + }); + + it('lists every check in verbose mode', () => { + const checks: DoctorCheck[] = [ + { id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0', group: 'toolchain' }, + ]; + + const output = formatDoctorReport({ title: 'KTX status', checks }, { verbose: true }); + expect(output).toContain('✓ Node 22+: v22.16.0'); }); }); @@ -127,6 +138,7 @@ describe('runSetupDoctorChecks', () => { status: 'fail', detail: 'pnpm not found', fix: 'Run: corepack enable && corepack prepare pnpm@10.28.0 --activate', + group: 'toolchain', }); expect(checks).toContainEqual({ id: 'package-build', @@ -134,6 +146,7 @@ describe('runSetupDoctorChecks', () => { status: 'fail', detail: 'Missing packages/cli/dist/bin.js', fix: 'Run: pnpm run build', + group: 'toolchain', }); }); @@ -154,9 +167,11 @@ describe('runSetupDoctorChecks', () => { const testIo = makeIo(); await expect( - runKtxDoctor({ command: 'setup', outputMode: 'plain', inputMode: 'disabled' }, testIo.io, { - runSetupChecks: async () => checks, - }), + runKtxDoctor( + { command: 'setup', outputMode: 'plain', inputMode: 'disabled', verbose: true }, + testIo.io, + { runSetupChecks: async () => checks }, + ), ).resolves.toBe(0); expect(checks).toContainEqual({ @@ -165,8 +180,9 @@ describe('runSetupDoctorChecks', () => { status: 'warn', detail: 'spawn corepack ENOENT', fix: 'Run: corepack enable', + group: 'toolchain', }); - expect(testIo.stdout()).toContain('WARN Corepack: spawn corepack ENOENT'); + expect(testIo.stdout()).toContain('⚠ Corepack: spawn corepack ENOENT'); expect(testIo.stderr()).toBe(''); }); }); @@ -204,12 +220,45 @@ describe('runKtxDoctor', () => { ), ).resolves.toBe(1); - expect(testIo.stdout()).toContain('KTX setup doctor'); - expect(testIo.stdout()).toContain('FAIL TypeScript package build: Missing packages/cli/dist/bin.js'); - expect(testIo.stdout()).toContain('Fix: Run: pnpm run build'); + expect(testIo.stdout()).toContain('KTX status'); + expect(testIo.stdout()).toContain('No project here yet.'); + expect(testIo.stdout()).toContain('Before you can run'); + expect(testIo.stdout()).toContain('✗ TypeScript package build: Missing packages/cli/dist/bin.js'); + expect(testIo.stdout()).toContain('→ Run: pnpm run build'); expect(testIo.stderr()).toBe(''); }); + it('leads with `ktx setup` and hides toolchain warnings when no project exists', async () => { + const testIo = makeIo(); + + await expect( + runKtxDoctor( + { command: 'setup', outputMode: 'plain', inputMode: 'disabled' }, + testIo.io, + { + runSetupChecks: async () => [ + { id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0', group: 'toolchain' }, + { + id: 'corepack', + label: 'Corepack', + status: 'warn', + detail: 'spawn corepack ENOENT', + fix: 'Run: corepack enable', + group: 'toolchain', + }, + ], + }, + ), + ).resolves.toBe(0); + + const out = testIo.stdout(); + expect(out).toContain('No project here yet.'); + expect(out).toContain('Run'); + expect(out).toContain('ktx setup'); + expect(out).not.toContain('Corepack'); + expect(out).not.toContain('Node 22+'); + }); + it('prints JSON setup report', async () => { const testIo = makeIo(); @@ -226,12 +275,13 @@ describe('runKtxDoctor', () => { ).resolves.toBe(0); expect(JSON.parse(testIo.stdout())).toEqual({ - title: 'KTX setup doctor', + title: 'KTX status', checks: [{ id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' }], }); }); it('runs project checks against a valid ktx.yaml', async () => { + process.env.ANTHROPIC_API_KEY = 'test-key'; await writeFile( join(tempDir, 'ktx.yaml'), [ @@ -240,220 +290,109 @@ describe('runKtxDoctor', () => { ' warehouse:', ' driver: sqlite', ' path: ./warehouse.db', + 'llm:', + ' provider:', + ' backend: anthropic', + ' models:', + ' default: claude-sonnet-4-5', 'ingest:', ' adapters:', ' - live-database', + ' embeddings:', + ' backend: openai', + ' model: text-embedding-3-small', + ' dimensions: 1536', '', ].join('\n'), 'utf-8', ); + process.env.OPENAI_API_KEY = 'test-key'; const testIo = makeIo(); await expect( runKtxDoctor( { command: 'project', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' }, testIo.io, - { - runSetupChecks: async () => [ - { id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' }, - ], - }, + {}, ), ).resolves.toBe(0); - expect(testIo.stdout()).toContain('KTX project doctor'); - expect(testIo.stdout()).toContain('PASS Project config: warehouse'); - expect(testIo.stdout()).toContain('PASS Connections: 1 configured'); + const out = testIo.stdout(); + expect(out).toContain('KTX status'); + expect(out).toContain('· warehouse'); + expect(out).toContain('Connections (1)'); + expect(out).toContain('LLM'); + expect(out).toContain('anthropic'); + expect(out).toContain('Embeddings'); + expect(out).toContain('Ready.'); + delete process.env.ANTHROPIC_API_KEY; + delete process.env.OPENAI_API_KEY; }); - it('includes Postgres historic-SQL readiness in project doctor output', async () => { + it('returns blocked verdict when LLM is not configured', async () => { await writeFile( join(tempDir, 'ktx.yaml'), [ 'project: warehouse', 'connections:', ' warehouse:', - ' driver: postgres', - ' url: env:WAREHOUSE_DATABASE_URL', - ' historicSql:', - ' enabled: true', - ' dialect: postgres', - 'ingest:', - ' adapters:', - ' - live-database', - ' - historic-sql', + ' driver: sqlite', + ' path: ./warehouse.db', '', ].join('\n'), 'utf-8', ); const testIo = makeIo(); - const runHistoricSqlDoctorChecks = vi.fn(async () => [ - { - id: 'historic-sql-postgres-warehouse', - label: 'Postgres Historic SQL (warehouse)', - status: 'pass' as const, - detail: - 'pg_stat_statements ready (PostgreSQL 16.4); info: pg_stat_statements.max is 1000; set it to at least 5000 to reduce query-template eviction churn', - }, - ]); await expect( runKtxDoctor( { command: 'project', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' }, testIo.io, - { - runSetupChecks: async () => [ - { id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' }, - ], - runHistoricSqlDoctorChecks, - }, + {}, ), - ).resolves.toBe(0); + ).resolves.toBe(1); - expect(runHistoricSqlDoctorChecks).toHaveBeenCalledTimes(1); - expect(testIo.stdout()).toContain('PASS Postgres Historic SQL (warehouse): pg_stat_statements ready'); - expect(testIo.stdout()).toContain('info: pg_stat_statements.max is 1000'); - expect(testIo.stdout()).not.toContain('Fix: Update the Postgres parameter group or config'); + expect(testIo.stdout()).toContain('no LLM configured'); + expect(testIo.stdout()).toContain('ktx setup'); }); it('warns when semantic-search embeddings are not configured', async () => { - await writeProjectConfig(tempDir, ['backend: deterministic', 'model: deterministic', 'dimensions: 8']); + process.env.ANTHROPIC_API_KEY = 'test-key'; + await writeFile( + join(tempDir, 'ktx.yaml'), + [ + 'project: warehouse', + 'connections:', + ' warehouse:', + ' driver: sqlite', + ' path: ./warehouse.db', + 'llm:', + ' provider:', + ' backend: anthropic', + 'ingest:', + ' adapters:', + ' - live-database', + ' embeddings:', + ' backend: deterministic', + ' model: deterministic', + ' dimensions: 8', + '', + ].join('\n'), + 'utf-8', + ); const testIo = makeIo(); await expect( runKtxDoctor( { command: 'project', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' }, testIo.io, - { - runSetupChecks: async () => [ - { id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' }, - ], - }, + {}, ), ).resolves.toBe(0); - expect(testIo.stdout()).toContain('WARN Semantic search embeddings: ingest.embeddings.backend is deterministic.'); - expect(testIo.stdout()).toContain( - 'Semantic lane will be skipped; lexical, dictionary, and token lanes remain available.', - ); - expect(testIo.stdout()).toContain( - `Fix: Run: ktx setup --project-dir ${tempDir} --no-input`, - ); - }); - - it('probes configured semantic-search embeddings for project doctor', async () => { - await writeProjectConfig(tempDir, [ - 'backend: sentence-transformers', - 'model: all-MiniLM-L6-v2', - 'dimensions: 384', - 'sentenceTransformers:', - ' base_url: http://127.0.0.1:8765', - " pathPrefix: ''", - ]); - const healthCheck = vi.fn(async () => ({ ok: true })); - const testIo = makeIo(); - - await expect( - runKtxDoctor( - { command: 'project', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' }, - testIo.io, - { - runSetupChecks: async () => [ - { id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' }, - ], - embeddingHealthCheck: healthCheck, - embeddingProbeTimeoutMs: 1234, - }, - ), - ).resolves.toBe(0); - - expect(healthCheck).toHaveBeenCalledWith( - { - backend: 'sentence-transformers', - model: 'all-MiniLM-L6-v2', - dimensions: 384, - sentenceTransformers: { baseURL: 'http://127.0.0.1:8765', pathPrefix: '' }, - }, - { text: 'KTX semantic search doctor probe', timeoutMs: 1234 }, - ); - expect(testIo.stdout()).toContain( - 'PASS Semantic search embeddings: sentence-transformers/all-MiniLM-L6-v2 (384d) probe succeeded', - ); - }); - - it('allows local sentence-transformers semantic-search probes enough time for cold start', async () => { - await writeProjectConfig(tempDir, [ - 'backend: sentence-transformers', - 'model: all-MiniLM-L6-v2', - 'dimensions: 384', - 'sentenceTransformers:', - ' base_url: http://127.0.0.1:8765', - " pathPrefix: ''", - ]); - const healthCheck = vi.fn(async () => ({ ok: true })); - const testIo = makeIo(); - - await expect( - runKtxDoctor( - { command: 'project', projectDir: tempDir, outputMode: 'plain', inputMode: 'disabled' }, - testIo.io, - { - runSetupChecks: async () => [ - { id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' }, - ], - embeddingHealthCheck: healthCheck, - }, - ), - ).resolves.toBe(0); - - expect(healthCheck).toHaveBeenCalledWith( - expect.objectContaining({ - backend: 'sentence-transformers', - model: 'all-MiniLM-L6-v2', - dimensions: 384, - }), - { text: 'KTX semantic search doctor probe', timeoutMs: 120_000 }, - ); - }); - - it('reports unhealthy semantic-search embeddings as a warning in JSON output', async () => { - await writeProjectConfig(tempDir, [ - 'backend: sentence-transformers', - 'model: all-MiniLM-L6-v2', - 'dimensions: 384', - 'sentenceTransformers:', - ' base_url: http://127.0.0.1:8765', - " pathPrefix: ''", - ]); - const healthCheck = vi.fn(async () => ({ - ok: false, - message: 'connect ECONNREFUSED 127.0.0.1:8765', - })); - const testIo = makeIo(); - - await expect( - runKtxDoctor( - { command: 'project', projectDir: tempDir, outputMode: 'json', inputMode: 'disabled' }, - testIo.io, - { - runSetupChecks: async () => [ - { id: 'node', label: 'Node 22+', status: 'pass', detail: 'v22.16.0 ABI 127' }, - ], - embeddingHealthCheck: healthCheck, - }, - ), - ).resolves.toBe(0); - - const report = JSON.parse(testIo.stdout()) as { - checks: Array<{ id: string; label: string; status: string; detail: string; fix?: string }>; - }; - expect(report.checks).toContainEqual({ - id: 'semantic-search-embeddings', - label: 'Semantic search embeddings', - status: 'warn', - detail: - 'sentence-transformers/all-MiniLM-L6-v2 (384d) probe failed: connect ECONNREFUSED 127.0.0.1:8765. Semantic lane will be skipped; lexical, dictionary, and token lanes remain available.', - fix: `Run: ktx setup --project-dir ${tempDir} --no-input`, - }); + expect(testIo.stdout()).toContain('Embeddings'); + expect(testIo.stdout()).toContain('deterministic'); + expect(testIo.stdout()).toContain('semantic search degraded'); + delete process.env.ANTHROPIC_API_KEY; }); }); diff --git a/packages/cli/src/doctor.ts b/packages/cli/src/doctor.ts index 4203369c..9342f24e 100644 --- a/packages/cli/src/doctor.ts +++ b/packages/cli/src/doctor.ts @@ -4,15 +4,13 @@ import { access } from 'node:fs/promises'; import { join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { promisify } from 'node:util'; -import type { KtxLocalProject, KtxProjectEmbeddingConfig } from '@ktx/context/project'; -import type { KtxEmbeddingConfig, KtxEmbeddingHealthCheckOptions, KtxEmbeddingHealthCheckResult } from '@ktx/llm'; -import type { HistoricSqlDoctorDeps } from './historic-sql-doctor.js'; const execFileAsync = promisify(execFile); type DoctorStatus = 'pass' | 'warn' | 'fail'; type KtxDoctorOutputMode = 'plain' | 'json'; type KtxDoctorInputMode = 'auto' | 'disabled'; +type DoctorGroup = 'toolchain' | 'project' | 'search' | 'history'; export interface DoctorCheck { id: string; @@ -20,6 +18,7 @@ export interface DoctorCheck { status: DoctorStatus; detail: string; fix?: string; + group?: DoctorGroup; } interface DoctorReport { @@ -28,11 +27,22 @@ interface DoctorReport { } export type KtxDoctorArgs = - | { command: 'setup'; outputMode: KtxDoctorOutputMode; inputMode?: KtxDoctorInputMode } - | { command: 'project'; projectDir: string; outputMode: KtxDoctorOutputMode; inputMode?: KtxDoctorInputMode }; + | { + command: 'setup'; + outputMode: KtxDoctorOutputMode; + inputMode?: KtxDoctorInputMode; + verbose?: boolean; + } + | { + command: 'project'; + projectDir: string; + outputMode: KtxDoctorOutputMode; + inputMode?: KtxDoctorInputMode; + verbose?: boolean; + }; interface KtxDoctorIo { - stdout: { write(chunk: string): void }; + stdout: { isTTY?: boolean; write(chunk: string): void }; stderr: { write(chunk: string): void }; } @@ -44,20 +54,8 @@ interface SetupDoctorDeps { importBetterSqlite3?: () => Promise; } -type EmbeddingHealthCheck = ( - config: KtxEmbeddingConfig, - options?: KtxEmbeddingHealthCheckOptions, -) => Promise; - -interface SemanticSearchDoctorDeps { - env?: NodeJS.ProcessEnv; - embeddingHealthCheck?: EmbeddingHealthCheck; - embeddingProbeTimeoutMs?: number; -} - -interface KtxDoctorDeps extends SemanticSearchDoctorDeps, HistoricSqlDoctorDeps { +interface KtxDoctorDeps { runSetupChecks?: () => Promise; - runHistoricSqlDoctorChecks?: (project: KtxLocalProject, deps: HistoricSqlDoctorDeps) => Promise; } function workspaceRootDir(): string { @@ -118,99 +116,6 @@ function check(status: DoctorStatus, id: string, label: string, detail: string, return fix ? { id, label, status, detail, fix } : { id, label, status, detail }; } -const SEMANTIC_SEARCH_HEALTH_TEXT = 'KTX semantic search doctor probe'; -const SEMANTIC_SEARCH_HEALTH_TIMEOUT_MS = 5_000; -const SEMANTIC_SEARCH_LOCAL_HEALTH_TIMEOUT_MS = 120_000; - -function semanticEmbeddingSetupFix(projectDir: string, backend: KtxProjectEmbeddingConfig['backend']): string { - if (backend === 'openai') { - return `Set OPENAI_API_KEY or rerun: ktx setup --project-dir ${projectDir} --embedding-backend openai --no-input`; - } - return `Run: ktx setup --project-dir ${projectDir} --no-input`; -} - -function embeddingConfigLabel(config: KtxProjectEmbeddingConfig | KtxEmbeddingConfig): string { - const model = config.model?.trim() || 'model not configured'; - return `${config.backend}/${model} (${config.dimensions}d)`; -} - -function semanticLaneFallbackDetail(reason: string): string { - return `${reason}. Semantic lane will be skipped; lexical, dictionary, and token lanes remain available.`; -} - -async function defaultEmbeddingHealthCheck( - config: KtxEmbeddingConfig, - options?: KtxEmbeddingHealthCheckOptions, -): Promise { - const { runKtxEmbeddingHealthCheck } = await import('@ktx/llm'); - return runKtxEmbeddingHealthCheck(config, options); -} - -async function runSemanticSearchEmbeddingCheck( - config: KtxProjectEmbeddingConfig, - projectDir: string, - deps: SemanticSearchDoctorDeps = {}, -): Promise { - if (config.backend === 'none' || config.backend === 'deterministic') { - return check( - 'warn', - 'semantic-search-embeddings', - 'Semantic search embeddings', - semanticLaneFallbackDetail(`ingest.embeddings.backend is ${config.backend}`), - semanticEmbeddingSetupFix(projectDir, config.backend), - ); - } - - try { - const { resolveLocalKtxEmbeddingConfig } = await import('@ktx/context'); - const resolved = resolveLocalKtxEmbeddingConfig(config, deps.env ?? process.env); - if (!resolved) { - return check( - 'warn', - 'semantic-search-embeddings', - 'Semantic search embeddings', - semanticLaneFallbackDetail(`No runtime embedding config resolved for ${embeddingConfigLabel(config)}`), - semanticEmbeddingSetupFix(projectDir, config.backend), - ); - } - - const healthCheck = deps.embeddingHealthCheck ?? defaultEmbeddingHealthCheck; - const timeoutMs = - deps.embeddingProbeTimeoutMs ?? - (resolved.backend === 'sentence-transformers' - ? SEMANTIC_SEARCH_LOCAL_HEALTH_TIMEOUT_MS - : SEMANTIC_SEARCH_HEALTH_TIMEOUT_MS); - const health = await healthCheck(resolved, { - text: SEMANTIC_SEARCH_HEALTH_TEXT, - timeoutMs, - }); - if (health.ok) { - return check( - 'pass', - 'semantic-search-embeddings', - 'Semantic search embeddings', - `${embeddingConfigLabel(resolved)} probe succeeded`, - ); - } - - return check( - 'warn', - 'semantic-search-embeddings', - 'Semantic search embeddings', - semanticLaneFallbackDetail(`${embeddingConfigLabel(resolved)} probe failed: ${health.message}`), - semanticEmbeddingSetupFix(projectDir, config.backend), - ); - } catch (error) { - return check( - 'warn', - 'semantic-search-embeddings', - 'Semantic search embeddings', - semanticLaneFallbackDetail(`${embeddingConfigLabel(config)} probe failed: ${failureMessage(error)}`), - semanticEmbeddingSetupFix(projectDir, config.backend), - ); - } -} - export async function runSetupDoctorChecks(deps: SetupDoctorDeps = {}): Promise { const env = deps.env ?? process.env; const root = deps.workspaceRoot ?? workspaceRootDir(); @@ -304,56 +209,231 @@ export async function runSetupDoctorChecks(deps: SetupDoctorDeps = {}): Promise< ); } - return checks; + return checks.map((entry) => ({ ...entry, group: 'toolchain' })); } -async function runProjectChecks(projectDir: string, deps: KtxDoctorDeps = {}): Promise { - const { loadKtxProject } = await import('@ktx/context/project'); - const checks: DoctorCheck[] = []; - try { - const project = await loadKtxProject({ projectDir }); - checks.push(check('pass', 'project-config', 'Project config', project.config.project)); - const connectionCount = Object.keys(project.config.connections).length; - checks.push( - connectionCount > 0 - ? check('pass', 'connections', 'Connections', `${connectionCount} configured`) - : check( - 'warn', - 'connections', - 'Connections', - '0 configured', - 'Add a connection to ktx.yaml or run `ktx setup`', - ), - ); - checks.push(check('pass', 'storage', 'Storage', `${project.config.storage.state}/${project.config.storage.search}`)); - checks.push(check('pass', 'llm-provider', 'LLM provider', project.config.llm.provider.backend)); - checks.push(await runSemanticSearchEmbeddingCheck(project.config.ingest.embeddings, projectDir, deps)); - const runHistoricSqlDoctorChecks = - deps.runHistoricSqlDoctorChecks ?? (await import('./historic-sql-doctor.js')).runPostgresHistoricSqlDoctorChecks; - checks.push(...(await runHistoricSqlDoctorChecks(project, deps))); - } catch (error) { - checks.push( - check( - 'fail', - 'project-config', - 'Project config', - failureMessage(error), - `Run: ktx init ${projectDir} --name `, - ), - ); +const STATUS_SYMBOL: Record = { pass: '✓', warn: '⚠', fail: '✗' }; + +const GROUP_ORDER: DoctorGroup[] = ['toolchain', 'project', 'search', 'history']; + +const GROUP_LABEL: Record = { + toolchain: 'Environment', + project: 'Project', + search: 'Semantic search', + history: 'Query history', +}; + +function shouldUseColor(io: KtxDoctorIo): boolean { + if (io.stdout.isTTY !== true) return false; + const env = process.env; + return !env.NO_COLOR && env.TERM !== 'dumb' && !env.CI; +} + +function styleStatus(useColor: boolean, status: DoctorStatus, text: string): string { + if (!useColor) return text; + const code = status === 'pass' ? 32 : status === 'warn' ? 33 : 31; + return `\u001b[${code}m${text}\u001b[39m`; +} + +function styleDim(useColor: boolean, text: string): string { + return useColor ? `\u001b[2m${text}\u001b[22m` : text; +} + +function styleBold(useColor: boolean, text: string): string { + return useColor ? `\u001b[1m${text}\u001b[22m` : text; +} + +function groupOf(entry: DoctorCheck): DoctorGroup { + return entry.group ?? 'project'; +} + +function aggregateStatus(checks: DoctorCheck[]): DoctorStatus { + if (checks.some((c) => c.status === 'fail')) return 'fail'; + if (checks.some((c) => c.status === 'warn')) return 'warn'; + return 'pass'; +} + +function abbreviateHome(filePath: string | undefined): string | undefined { + if (!filePath) return filePath; + const home = process.env.HOME; + if (home && (filePath === home || filePath.startsWith(`${home}/`))) { + return filePath === home ? '~' : `~${filePath.slice(home.length)}`; } - return checks; + return filePath; } -export function formatDoctorReport(report: DoctorReport): string { - const lines = [report.title]; - for (const item of report.checks) { - lines.push(`${item.status.toUpperCase()} ${item.label}: ${item.detail}`); - if (item.fix) { - lines.push(` Fix: ${item.fix}`); +function groupSummaryWhenAllPass(entries: DoctorCheck[]): string { + if (entries.length === 1) { + const only = entries[0]!; + return only.detail || only.label; + } + return entries.map((c) => c.label).join(' · '); +} + +interface RenderOptions { + verbose: boolean; + useColor: boolean; + durationMs?: number; + projectName?: string; + projectDir?: string; + command?: 'setup' | 'project'; +} + +const NEXT_STEPS_PROJECT = ['ktx scan', 'ktx wiki', 'ktx sl ask "…"']; + +export function formatDoctorReport(report: DoctorReport, options: Partial = {}): string { + const opts: RenderOptions = { + verbose: options.verbose ?? false, + useColor: options.useColor ?? false, + durationMs: options.durationMs, + projectName: options.projectName, + projectDir: options.projectDir, + command: options.command, + }; + return renderPlainReport(report, opts); +} + +function renderSetupReport(report: DoctorReport, options: RenderOptions): string { + const { verbose, useColor } = options; + const dim = (text: string) => styleDim(useColor, text); + const bold = (text: string) => styleBold(useColor, text); + const status = (s: DoctorStatus, text: string) => styleStatus(useColor, s, text); + const symbol = (s: DoctorStatus) => status(s, STATUS_SYMBOL[s]); + + const fails = report.checks.filter((c) => c.status === 'fail'); + const lines: string[] = []; + lines.push(bold(report.title)); + lines.push(''); + lines.push(` No project here yet.`); + lines.push(''); + + if (fails.length > 0) { + lines.push(` Before you can run ${bold('ktx setup')}, fix this:`); + for (const entry of fails) { + lines.push(` ${symbol('fail')} ${entry.label}: ${entry.detail}`); + if (entry.fix) { + lines.push(` ${dim(`→ ${entry.fix}`)}`); + } + } + lines.push(''); + } else { + lines.push(` Run ${bold('ktx setup')} to get started.`); + lines.push(''); + } + + if (verbose) { + lines.push(dim(' Toolchain:')); + for (const entry of report.checks) { + lines.push(` ${symbol(entry.status)} ${entry.label}: ${entry.detail}`); + if (entry.fix && entry.status !== 'pass') { + lines.push(` ${dim(`→ ${entry.fix}`)}`); + } + } + lines.push(''); + } + + return lines.join('\n'); +} + +function renderPlainReport(report: DoctorReport, options: RenderOptions): string { + if (options.command === 'setup') return renderSetupReport(report, options); + const { verbose, useColor, durationMs, projectName, projectDir } = options; + const dim = (text: string) => styleDim(useColor, text); + const bold = (text: string) => styleBold(useColor, text); + const status = (s: DoctorStatus, text: string) => styleStatus(useColor, s, text); + const symbol = (s: DoctorStatus) => status(s, STATUS_SYMBOL[s]); + + const lines: string[] = []; + const titleParts: string[] = [bold(report.title)]; + if (projectName) titleParts.push(projectName); + const abbreviatedDir = abbreviateHome(projectDir); + const titleLine = titleParts.join(` ${dim('·')} `); + const dirSuffix = abbreviatedDir ? ` ${dim(`(${abbreviatedDir})`)}` : ''; + lines.push(`${titleLine}${dirSuffix}`); + lines.push(''); + + const groups = new Map(); + for (const entry of report.checks) { + const group = groupOf(entry); + const bucket = groups.get(group) ?? []; + bucket.push(entry); + groups.set(group, bucket); + } + + const orderedGroups: DoctorGroup[] = []; + for (const g of GROUP_ORDER) { + if (groups.has(g)) orderedGroups.push(g); + } + for (const g of groups.keys()) { + if (!orderedGroups.includes(g)) orderedGroups.push(g); + } + + const labelWidth = orderedGroups.reduce( + (max, g) => Math.max(max, (GROUP_LABEL[g] ?? g).length), + 0, + ); + + for (const group of orderedGroups) { + const entries = groups.get(group) ?? []; + const head = aggregateStatus(entries); + const nonPass = entries.filter((c) => c.status !== 'pass'); + const label = (GROUP_LABEL[group] ?? group).padEnd(labelWidth); + + if (nonPass.length === 0) { + lines.push(` ${symbol(head)} ${label} ${dim(groupSummaryWhenAllPass(entries))}`); + if (verbose) { + for (const entry of entries) { + lines.push(` ${symbol(entry.status)} ${entry.label}: ${entry.detail}`); + } + } + continue; + } + + if (entries.length === 1) { + const only = entries[0]!; + lines.push(` ${symbol(only.status)} ${label} ${only.detail}`); + if (only.fix) { + lines.push(` ${' '.repeat(2 + labelWidth + 4)}${dim(`→ ${only.fix}`)}`); + } + continue; + } + + lines.push(` ${symbol(head)} ${label} ${dim(`${nonPass.length} of ${entries.length} need attention`)}`); + for (const entry of entries) { + if (entry.status === 'pass' && !verbose) continue; + lines.push(` ${symbol(entry.status)} ${entry.label}: ${entry.detail}`); + if (entry.fix) { + lines.push(` ${dim(`→ ${entry.fix}`)}`); + } } } + lines.push(''); + + const totalFail = report.checks.filter((c) => c.status === 'fail').length; + const totalWarn = report.checks.filter((c) => c.status === 'warn').length; + const durationText = durationMs !== undefined ? ` ${dim(`(${(durationMs / 1000).toFixed(2)}s)`)}` : ''; + + if (totalFail === 0 && totalWarn === 0) { + const hint = ` ${dim('Try:')} ${NEXT_STEPS_PROJECT.join(dim(' · '))}`; + lines.push(`${status('pass', 'Everything ready.')}${hint}${durationText}`); + } else if (totalFail === 0) { + const word = totalWarn === 1 ? 'warning' : 'warnings'; + lines.push( + `${status('warn', `${totalWarn} ${word}.`)} ${dim('Run')} ktx status --verbose ${dim('for full details.')}${durationText}`, + ); + } else { + const fWord = totalFail === 1 ? 'issue' : 'issues'; + const warnSuffix = + totalWarn > 0 + ? ` ${dim('·')} ${status('warn', `${totalWarn} ${totalWarn === 1 ? 'warning' : 'warnings'}`)}` + : ''; + lines.push( + `${status('fail', `${totalFail} ${fWord} to fix.`)}${warnSuffix}${durationText}`, + ); + } + lines.push(''); + return lines.join('\n'); } @@ -361,12 +441,12 @@ function hasFailures(report: DoctorReport): boolean { return report.checks.some((item) => item.status === 'fail'); } -function writeReport(report: DoctorReport, outputMode: KtxDoctorOutputMode, io: KtxDoctorIo): void { +function writeReport(report: DoctorReport, outputMode: KtxDoctorOutputMode, io: KtxDoctorIo, options: RenderOptions): void { if (outputMode === 'json') { io.stdout.write(`${JSON.stringify(report, null, 2)}\n`); return; } - io.stdout.write(formatDoctorReport(report)); + io.stdout.write(renderPlainReport(report, options)); } export async function runKtxDoctor( @@ -374,18 +454,41 @@ export async function runKtxDoctor( io: KtxDoctorIo = process, deps: KtxDoctorDeps = {}, ): Promise { + const startedAt = Date.now(); try { const runSetupChecks = deps.runSetupChecks ?? (() => runSetupDoctorChecks()); - const setupChecks = await runSetupChecks(); - const report: DoctorReport = - args.command === 'setup' - ? { title: 'KTX setup doctor', checks: setupChecks } - : { - title: 'KTX project doctor', - checks: [...setupChecks, ...(await runProjectChecks(args.projectDir, deps))], - }; - writeReport(report, args.outputMode, io); + if (args.command === 'project') { + const { loadKtxProject } = await import('@ktx/context/project'); + const { buildProjectStatus, renderProjectStatus } = await import('./status-project.js'); + const project = await loadKtxProject({ projectDir: args.projectDir }); + const projectStatus = buildProjectStatus(project); + const verbose = args.verbose ?? false; + const toolchainChecks = verbose ? await runSetupChecks() : undefined; + if (args.outputMode === 'json') { + io.stdout.write(`${JSON.stringify(projectStatus, null, 2)}\n`); + } else { + io.stdout.write( + renderProjectStatus(projectStatus, { + verbose, + useColor: shouldUseColor(io), + durationMs: Date.now() - startedAt, + toolchainChecks, + }), + ); + } + return projectStatus.verdict === 'blocked' ? 1 : 0; + } + + const setupChecks = await runSetupChecks(); + const report: DoctorReport = { title: 'KTX status', checks: setupChecks }; + const renderOptions: RenderOptions = { + verbose: args.verbose ?? false, + useColor: shouldUseColor(io), + durationMs: Date.now() - startedAt, + command: args.command, + }; + writeReport(report, args.outputMode, io, renderOptions); return hasFailures(report) ? 1 : 0; } catch (error) { io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); diff --git a/packages/cli/src/historic-sql-doctor.test.ts b/packages/cli/src/historic-sql-doctor.test.ts deleted file mode 100644 index f3bc347e..00000000 --- a/packages/cli/src/historic-sql-doctor.test.ts +++ /dev/null @@ -1,202 +0,0 @@ -import { buildDefaultKtxProjectConfig, type KtxProjectConnectionConfig } from '@ktx/context/project'; -import { HistoricSqlExtensionMissingError } from '@ktx/context/ingest'; -import { describe, expect, it, vi } from 'vitest'; -import { - runPostgresHistoricSqlDoctorChecks, - type HistoricSqlDoctorProject, - type PostgresHistoricSqlDoctorProbe, -} from './historic-sql-doctor.js'; - -function projectWithConnections(connections: Record): HistoricSqlDoctorProject { - return { - projectDir: '/tmp/ktx-project', - config: { - ...buildDefaultKtxProjectConfig('warehouse'), - connections, - ingest: { - ...buildDefaultKtxProjectConfig('warehouse').ingest, - adapters: ['live-database', 'historic-sql'], - }, - }, - }; -} - -describe('runPostgresHistoricSqlDoctorChecks', () => { - it('passes when no Postgres historic-SQL connections are enabled', async () => { - const checks = await runPostgresHistoricSqlDoctorChecks( - projectWithConnections({ - warehouse: { driver: 'sqlite', path: './warehouse.db' }, - }), - { - postgresHistoricSqlProbe: vi.fn(), - }, - ); - - expect(checks).toEqual([ - { - id: 'historic-sql-postgres', - label: 'Postgres Historic SQL', - status: 'pass', - detail: 'No enabled Postgres historic-SQL connections', - }, - ]); - }); - - it('passes when the PGSS probe succeeds without warnings', async () => { - const probe = vi.fn(async () => ({ - pgServerVersion: 'PostgreSQL 16.4', - warnings: [], - })); - - const checks = await runPostgresHistoricSqlDoctorChecks( - projectWithConnections({ - warehouse: { - driver: 'postgres', - url: 'env:WAREHOUSE_DATABASE_URL', - historicSql: { enabled: true, dialect: 'postgres' }, - }, - }), - { postgresHistoricSqlProbe: probe }, - ); - - expect(probe).toHaveBeenCalledWith({ - projectDir: '/tmp/ktx-project', - connectionId: 'warehouse', - connection: { - driver: 'postgres', - url: 'env:WAREHOUSE_DATABASE_URL', - historicSql: { enabled: true, dialect: 'postgres' }, - }, - env: process.env, - }); - expect(checks).toEqual([ - { - id: 'historic-sql-postgres-warehouse', - label: 'Postgres Historic SQL (warehouse)', - status: 'pass', - detail: 'pg_stat_statements ready (PostgreSQL 16.4)', - }, - ]); - }); - - it('passes with an informational note when only pg_stat_statements.max is below the recommended floor', async () => { - const checks = await runPostgresHistoricSqlDoctorChecks( - projectWithConnections({ - warehouse: { - driver: 'postgres', - url: 'env:WAREHOUSE_DATABASE_URL', - historicSql: { enabled: true, dialect: 'postgres' }, - }, - }), - { - postgresHistoricSqlProbe: async () => ({ - pgServerVersion: 'PostgreSQL 16.4', - warnings: [], - info: [ - 'pg_stat_statements.max is 1000; set it to at least 5000 to reduce query-template eviction churn', - ], - }), - }, - ); - - expect(checks).toEqual([ - { - id: 'historic-sql-postgres-warehouse', - label: 'Postgres Historic SQL (warehouse)', - status: 'pass', - detail: - 'pg_stat_statements ready (PostgreSQL 16.4); info: pg_stat_statements.max is 1000; set it to at least 5000 to reduce query-template eviction churn', - }, - ]); - }); - - it('warns when pg_stat_statements tracking is disabled', async () => { - const checks = await runPostgresHistoricSqlDoctorChecks( - projectWithConnections({ - warehouse: { - driver: 'postgres', - url: 'env:WAREHOUSE_DATABASE_URL', - historicSql: { enabled: true, dialect: 'postgres' }, - }, - }), - { - postgresHistoricSqlProbe: async () => ({ - pgServerVersion: 'PostgreSQL 16.4', - warnings: [ - 'pg_stat_statements.track is none; set it to top or all in the Postgres parameter group or config', - ], - info: [ - 'pg_stat_statements.max is 1000; set it to at least 5000 to reduce query-template eviction churn', - ], - }), - }, - ); - - expect(checks).toEqual([ - { - id: 'historic-sql-postgres-warehouse', - label: 'Postgres Historic SQL (warehouse)', - status: 'warn', - detail: - 'pg_stat_statements ready (PostgreSQL 16.4) with warnings: pg_stat_statements.track is none; set it to top or all in the Postgres parameter group or config; info: pg_stat_statements.max is 1000; set it to at least 5000 to reduce query-template eviction churn', - fix: 'Update the Postgres parameter group or config, then rerun `ktx status --project-dir /tmp/ktx-project`', - }, - ]); - }); - - it('fails when a connection has postgres historic SQL but is not a Postgres driver', async () => { - const checks = await runPostgresHistoricSqlDoctorChecks( - projectWithConnections({ - warehouse: { - driver: 'mysql', - url: 'env:WAREHOUSE_DATABASE_URL', - historicSql: { enabled: true, dialect: 'postgres' }, - }, - }), - { - postgresHistoricSqlProbe: vi.fn(), - }, - ); - - expect(checks).toEqual([ - { - id: 'historic-sql-postgres-warehouse', - label: 'Postgres Historic SQL (warehouse)', - status: 'fail', - detail: 'connections.warehouse.historicSql.dialect is postgres but driver is mysql', - fix: 'Set connections.warehouse.driver to postgres or disable historicSql for this connection', - }, - ]); - }); - - it('maps PGSS capability errors to actionable failures', async () => { - const checks = await runPostgresHistoricSqlDoctorChecks( - projectWithConnections({ - warehouse: { - driver: 'postgres', - url: 'env:WAREHOUSE_DATABASE_URL', - historicSql: { enabled: true, dialect: 'postgres' }, - }, - }), - { - postgresHistoricSqlProbe: async () => { - throw new HistoricSqlExtensionMissingError({ - dialect: 'postgres', - message: 'pg_stat_statements extension is not installed in the connection database.', - remediation: 'Run CREATE EXTENSION pg_stat_statements; against the connection database.', - }); - }, - }, - ); - - expect(checks).toEqual([ - { - id: 'historic-sql-postgres-warehouse', - label: 'Postgres Historic SQL (warehouse)', - status: 'fail', - detail: 'pg_stat_statements extension is not installed in the connection database.', - fix: 'Run CREATE EXTENSION pg_stat_statements; against the connection database.', - }, - ]); - }); -}); diff --git a/packages/cli/src/historic-sql-doctor.ts b/packages/cli/src/historic-sql-doctor.ts deleted file mode 100644 index bb9a681c..00000000 --- a/packages/cli/src/historic-sql-doctor.ts +++ /dev/null @@ -1,167 +0,0 @@ -import type { KtxProjectConfig, KtxProjectConnectionConfig } from '@ktx/context/project'; -import type { DoctorCheck } from './doctor.js'; - -export interface HistoricSqlDoctorProject { - projectDir: string; - config: Pick; -} - -export interface PostgresHistoricSqlDoctorProbeInput { - projectDir: string; - connectionId: string; - connection: KtxProjectConnectionConfig; - env: NodeJS.ProcessEnv; -} - -export interface PostgresHistoricSqlDoctorProbeResult { - pgServerVersion: string; - warnings: string[]; - info?: string[]; -} - -export type PostgresHistoricSqlDoctorProbe = ( - input: PostgresHistoricSqlDoctorProbeInput, -) => Promise; - -export interface HistoricSqlDoctorDeps { - env?: NodeJS.ProcessEnv; - postgresHistoricSqlProbe?: PostgresHistoricSqlDoctorProbe; -} - -function check(status: DoctorCheck['status'], id: string, label: string, detail: string, fix?: string): DoctorCheck { - return fix ? { id, label, status, detail, fix } : { id, label, status, detail }; -} - -function historicSqlRecord(connection: KtxProjectConnectionConfig): Record | null { - const historicSql = connection.historicSql; - return historicSql && typeof historicSql === 'object' && !Array.isArray(historicSql) - ? (historicSql as Record) - : null; -} - -function isEnabledPostgresHistoricSql(connection: KtxProjectConnectionConfig): boolean { - const historicSql = historicSqlRecord(connection); - return historicSql?.enabled === true && historicSql.dialect === 'postgres'; -} - -function isPostgresDriver(connection: KtxProjectConnectionConfig): boolean { - const driver = String(connection.driver ?? '').toLowerCase(); - return driver === 'postgres' || driver === 'postgresql'; -} - -function checkId(connectionId: string): string { - return `historic-sql-postgres-${connectionId.replace(/[^a-z0-9_-]+/gi, '-')}`; -} - -function capabilityFailureFix(error: unknown, connectionId: string, projectDir: string): string { - if (error instanceof Error && error.name === 'HistoricSqlExtensionMissingError' && 'remediation' in error) { - return String(error.remediation); - } - if (error instanceof Error && error.name === 'HistoricSqlGrantsMissingError' && 'remediation' in error) { - return String(error.remediation); - } - if (error instanceof Error && error.name === 'HistoricSqlVersionUnsupportedError') { - return 'Use PostgreSQL 14 or newer, or disable historicSql for this connection'; - } - return `Fix connections.${connectionId} Postgres settings, then rerun \`ktx status --project-dir ${projectDir}\``; -} - -function failureDetail(error: unknown): string { - if (error instanceof Error && error.message.trim().length > 0) { - return error.message.trim().split('\n')[0] ?? error.message.trim(); - } - return String(error); -} - -function readinessDetail(result: PostgresHistoricSqlDoctorProbeResult): string { - const warningText = result.warnings.length > 0 ? ` with warnings: ${result.warnings.join('; ')}` : ''; - const info = result.info ?? []; - const infoText = info.length > 0 ? `; info: ${info.join('; ')}` : ''; - return `pg_stat_statements ready (${result.pgServerVersion})${warningText}${infoText}`; -} - -async function defaultPostgresHistoricSqlProbe( - input: PostgresHistoricSqlDoctorProbeInput, -): Promise { - const [{ PostgresPgssReader }, { KtxPostgresHistoricSqlQueryClient, isKtxPostgresConnectionConfig }] = - await Promise.all([import('@ktx/context/ingest'), import('@ktx/connector-postgres')]); - - const inputDriver = input.connection.driver ?? 'unknown'; - if (!isKtxPostgresConnectionConfig(input.connection)) { - throw new Error(`Native PostgreSQL connector cannot run driver "${inputDriver}"`); - } - - const client = new KtxPostgresHistoricSqlQueryClient({ - connectionId: input.connectionId, - connection: input.connection, - env: input.env, - }); - try { - return await new PostgresPgssReader().probe(client); - } finally { - await client.cleanup(); - } -} - -export async function runPostgresHistoricSqlDoctorChecks( - project: HistoricSqlDoctorProject, - deps: HistoricSqlDoctorDeps = {}, -): Promise { - const targets = Object.entries(project.config.connections) - .filter(([, connection]) => isEnabledPostgresHistoricSql(connection)) - .sort(([left], [right]) => left.localeCompare(right)); - - if (targets.length === 0) { - return [ - check('pass', 'historic-sql-postgres', 'Postgres Historic SQL', 'No enabled Postgres historic-SQL connections'), - ]; - } - - const probe = deps.postgresHistoricSqlProbe ?? defaultPostgresHistoricSqlProbe; - const env = deps.env ?? process.env; - const checks: DoctorCheck[] = []; - for (const [connectionId, connection] of targets) { - const label = `Postgres Historic SQL (${connectionId})`; - if (!isPostgresDriver(connection)) { - checks.push( - check( - 'fail', - checkId(connectionId), - label, - `connections.${connectionId}.historicSql.dialect is postgres but driver is ${String(connection.driver)}`, - `Set connections.${connectionId}.driver to postgres or disable historicSql for this connection`, - ), - ); - continue; - } - - try { - const result = await probe({ projectDir: project.projectDir, connectionId, connection, env }); - if (result.warnings.length > 0) { - checks.push( - check( - 'warn', - checkId(connectionId), - label, - readinessDetail(result), - `Update the Postgres parameter group or config, then rerun \`ktx status --project-dir ${project.projectDir}\``, - ), - ); - } else { - checks.push(check('pass', checkId(connectionId), label, readinessDetail(result))); - } - } catch (error) { - checks.push( - check( - 'fail', - checkId(connectionId), - label, - failureDetail(error), - capabilityFailureFix(error, connectionId, project.projectDir), - ), - ); - } - } - - return checks; -} diff --git a/packages/cli/src/index.test.ts b/packages/cli/src/index.test.ts index 305cf30e..d1c2587e 100644 --- a/packages/cli/src/index.test.ts +++ b/packages/cli/src/index.test.ts @@ -1012,7 +1012,7 @@ describe('runKtxCli', () => { expect(setup).not.toHaveBeenCalled(); expect(doctor).toHaveBeenCalledWith( - { command: 'project', projectDir: tempDir, outputMode: 'json', inputMode: 'disabled' }, + { command: 'project', projectDir: tempDir, outputMode: 'json', inputMode: 'disabled', verbose: false }, statusIo.io, ); expect(statusIo.stderr()).toBe(''); @@ -1035,7 +1035,7 @@ describe('runKtxCli', () => { await expect(runKtxCli(['status', '--json', '--no-input'], statusIo.io, { doctor })).resolves.toBe(0); expect(doctor).toHaveBeenCalledWith( - { command: 'setup', outputMode: 'json', inputMode: 'disabled' }, + { command: 'setup', outputMode: 'json', inputMode: 'disabled', verbose: false }, statusIo.io, ); expect(statusIo.stderr()).toBe(''); diff --git a/packages/cli/src/status-project.ts b/packages/cli/src/status-project.ts new file mode 100644 index 00000000..47b9a49f --- /dev/null +++ b/packages/cli/src/status-project.ts @@ -0,0 +1,614 @@ +import type { + KtxLocalProject, + KtxProjectConfig, + KtxProjectConnectionConfig, + KtxProjectEmbeddingConfig, + KtxProjectLlmConfig, +} from '@ktx/context/project'; +import type { DoctorCheck } from './doctor.js'; + +type ProjectStatusLevel = 'ok' | 'warn' | 'fail'; +type ProjectVerdict = 'ready' | 'partial' | 'blocked'; + +interface ProjectStatusLine { + status: ProjectStatusLevel; + detail: string; + fix?: string; +} + +interface LlmStatus extends ProjectStatusLine { + backend: string; + model?: string; +} + +interface EmbeddingsStatus extends ProjectStatusLine { + backend: string; + model?: string; + dimensions?: number; +} + +interface ConnectionStatus extends ProjectStatusLine { + name: string; + driver: string; +} + +interface PipelineStatus { + adapters: string[]; + enrichmentMode: string; + relationshipsEnabled: boolean; + relationshipsLlmProposals: boolean; + relationshipsValidationRequired: boolean; + agentEnabled: boolean; + agentTools: string[]; + agentMaxIterations: number; +} + +interface StorageStatus { + state: string; + search: string; + gitAutoCommit: boolean; + gitAuthor: string; +} + +interface WarningItem { + message: string; + fix?: string; +} + +export interface ProjectStatus { + projectName: string; + projectDir: string; + llm: LlmStatus; + embeddings: EmbeddingsStatus; + storage: StorageStatus; + connections: ConnectionStatus[]; + pipeline: PipelineStatus; + warnings: WarningItem[]; + verdict: ProjectVerdict; + verdictReason: string; + nextActions: string[]; + promptCaching?: { enabled: boolean; systemTtl?: string; toolsTtl?: string; historyTtl?: string }; + workUnits?: { stepBudget: number; maxConcurrency: number; failureMode: string }; + memoryAutoCommit: boolean; + relationshipsDetail?: { + acceptThreshold: number; + reviewThreshold: number; + maxLlmTablesPerBatch: number; + validationConcurrency: number; + }; +} + +function resolveRef(value: unknown, env: NodeJS.ProcessEnv): { resolved: string; via: 'literal' | 'env' | 'file' | 'missing' } { + if (typeof value !== 'string') return { resolved: '', via: 'missing' }; + const trimmed = value.trim(); + if (trimmed.length === 0) return { resolved: '', via: 'missing' }; + if (trimmed.startsWith('env:')) { + const name = trimmed.slice(4).trim(); + const v = env[name]; + return v && v.trim().length > 0 ? { resolved: v, via: 'env' } : { resolved: '', via: 'missing' }; + } + if (trimmed.startsWith('file:')) { + return { resolved: trimmed.slice(5), via: 'file' }; + } + return { resolved: trimmed, via: 'literal' }; +} + +function envHint(value: unknown): string | undefined { + if (typeof value === 'string' && value.trim().startsWith('env:')) { + return value.trim().slice(4).trim(); + } + return undefined; +} + +function buildLlmStatus(config: KtxProjectLlmConfig, env: NodeJS.ProcessEnv): LlmStatus { + const backend = config.provider.backend; + const model = config.models?.default; + if (backend === 'none') { + return { + backend, + model, + status: 'fail', + detail: 'no LLM configured — ktx ask will not work', + fix: 'Run: ktx setup (choose an LLM provider)', + }; + } + if (backend === 'anthropic') { + const ref = config.provider.anthropic?.api_key; + const resolved = resolveRef(ref, env); + if (resolved.resolved.length > 0) { + return { backend, model, status: 'ok', detail: `key set${resolved.via === 'env' ? ` (env)` : ''}` }; + } + if (env.ANTHROPIC_API_KEY && env.ANTHROPIC_API_KEY.trim().length > 0) { + return { backend, model, status: 'ok', detail: 'key set (env: ANTHROPIC_API_KEY)' }; + } + const hint = envHint(ref); + return { + backend, + model, + status: 'warn', + detail: hint ? `key missing (env: ${hint})` : 'key missing', + fix: hint ? `Set ${hint}` : 'Set ANTHROPIC_API_KEY or rerun `ktx setup`', + }; + } + if (backend === 'vertex') { + const project = config.provider.vertex?.project; + if (project && project.length > 0) { + return { backend, model, status: 'ok', detail: `project=${project}` }; + } + return { backend, model, status: 'warn', detail: 'vertex project not configured', fix: 'Rerun `ktx setup`' }; + } + if (backend === 'gateway') { + const ref = config.provider.gateway?.api_key; + const resolved = resolveRef(ref, env); + if (resolved.resolved.length > 0) { + return { backend, model, status: 'ok', detail: 'key set' }; + } + const hint = envHint(ref); + return { + backend, + model, + status: 'warn', + detail: hint ? `key missing (env: ${hint})` : 'key missing', + fix: hint ? `Set ${hint}` : 'Set the gateway api_key or rerun `ktx setup`', + }; + } + return { backend, model, status: 'warn', detail: 'unknown LLM backend' }; +} + +function buildEmbeddingsStatus(config: KtxProjectEmbeddingConfig, env: NodeJS.ProcessEnv): EmbeddingsStatus { + const backend = config.backend; + const model = config.model; + const dimensions = config.dimensions; + if (backend === 'none') { + return { + backend, + model, + dimensions, + status: 'warn', + detail: 'disabled — semantic search will be skipped', + }; + } + if (backend === 'deterministic') { + return { + backend, + model, + dimensions, + status: 'warn', + detail: 'deterministic — semantic search degraded (lexical/dictionary lanes still work)', + }; + } + if (backend === 'openai') { + const ref = config.openai?.api_key; + const resolved = resolveRef(ref, env); + if (resolved.resolved.length > 0 || (env.OPENAI_API_KEY && env.OPENAI_API_KEY.trim().length > 0)) { + return { backend, model, dimensions, status: 'ok', detail: 'key set' }; + } + const hint = envHint(ref); + return { + backend, + model, + dimensions, + status: 'warn', + detail: hint ? `key missing (env: ${hint})` : 'key missing', + fix: hint ? `Set ${hint}` : 'Set OPENAI_API_KEY or rerun `ktx setup`', + }; + } + if (backend === 'sentence-transformers') { + const url = config.sentenceTransformers?.base_url; + if (typeof url === 'string' && url.length > 0) { + return { backend, model, dimensions, status: 'ok', detail: `service: ${url}` }; + } + return { + backend, + model, + dimensions, + status: 'warn', + detail: 'no base_url configured', + fix: 'Rerun `ktx setup`', + }; + } + return { backend, model, dimensions, status: 'warn', detail: 'unknown embedding backend' }; +} + +function buildConnectionStatus( + name: string, + conn: KtxProjectConnectionConfig, + env: NodeJS.ProcessEnv, +): ConnectionStatus { + const driver = (conn.driver ?? 'unknown').toLowerCase(); + const ok = (detail: string): ConnectionStatus => ({ name, driver, status: 'ok', detail }); + const warn = (detail: string, fix?: string): ConnectionStatus => ({ name, driver, status: 'warn', detail, fix }); + + switch (driver) { + case 'postgres': + case 'postgresql': + case 'mysql': + case 'clickhouse': + case 'sqlserver': { + const urlRef = resolveRef(conn.url, env); + if (urlRef.resolved.length > 0) return ok(`url configured`); + if (typeof (conn as Record).host === 'string') return ok('host configured'); + const hint = envHint(conn.url); + return warn(hint ? `url missing (env: ${hint})` : 'url not set', hint ? `Set ${hint}` : 'Rerun `ktx setup`'); + } + case 'snowflake': { + const account = (conn as Record).account; + if (typeof account === 'string' && account.length > 0) return ok(`account: ${account}`); + return warn('account not set', 'Rerun `ktx setup`'); + } + case 'bigquery': { + const cred = resolveRef((conn as Record).credentials_json, env); + if (cred.resolved.length > 0) return ok('credentials configured'); + const hint = envHint((conn as Record).credentials_json); + return warn(hint ? `credentials missing (env: ${hint})` : 'credentials not set', hint ? `Set ${hint}` : 'Rerun `ktx setup`'); + } + case 'sqlite': { + const path = (conn as Record).path; + if (typeof path === 'string' && path.length > 0) return ok(`path: ${path}`); + return warn('path not set', 'Rerun `ktx setup`'); + } + case 'notion': { + const tokenRef = + (conn as Record).auth_token_ref ?? + (conn as Record).auth_token; + const resolved = resolveRef(tokenRef, env); + if (resolved.resolved.length > 0) return ok('auth token configured'); + const hint = envHint(tokenRef); + return warn(hint ? `auth token missing (env: ${hint})` : 'auth token not set', hint ? `Set ${hint}` : 'Rerun `ktx setup`'); + } + case 'dbt': + case 'dbt-core': + case 'dbt-cloud': { + const repoUrl = + (conn as Record).repoUrl ?? + (conn as Record).repo_url; + if (typeof repoUrl === 'string' && repoUrl.length > 0) return ok(`repo: ${repoUrl}`); + return warn('repoUrl not set', 'Rerun `ktx setup`'); + } + case 'metabase': { + const url = (conn as Record).url ?? (conn as Record).base_url; + if (typeof url === 'string' && url.length > 0) return ok(`url: ${url}`); + return warn('url not set', 'Rerun `ktx setup`'); + } + case 'looker': + case 'lookml': { + const url = (conn as Record).base_url ?? (conn as Record).url; + if (typeof url === 'string' && url.length > 0) return ok(`url: ${url}`); + return warn('base_url not set', 'Rerun `ktx setup`'); + } + case 'metricflow': { + const repoUrl = (conn as Record).repoUrl ?? (conn as Record).repo_url; + if (typeof repoUrl === 'string' && repoUrl.length > 0) return ok(`repo: ${repoUrl}`); + return warn('repoUrl not set', 'Rerun `ktx setup`'); + } + default: + return { name, driver, status: 'ok', detail: 'configured' }; + } +} + +const ADAPTER_DRIVER_REQUIREMENT: Record = { + 'live-database': ['postgres', 'postgresql', 'mysql', 'snowflake', 'bigquery', 'clickhouse', 'sqlite', 'sqlserver'], + dbt: ['dbt', 'dbt-core', 'dbt-cloud'], + notion: ['notion'], + metabase: ['metabase'], + looker: ['looker', 'lookml'], + lookml: ['looker', 'lookml'], + metricflow: ['metricflow'], +}; + +function buildPipelineStatus(config: KtxProjectConfig): PipelineStatus { + return { + adapters: config.ingest.adapters, + enrichmentMode: config.scan.enrichment.mode, + relationshipsEnabled: config.scan.relationships.enabled, + relationshipsLlmProposals: config.scan.relationships.llmProposals, + relationshipsValidationRequired: config.scan.relationships.validationRequiredForManifest, + agentEnabled: config.agent.run_research.enabled, + agentTools: config.agent.run_research.default_toolset, + agentMaxIterations: config.agent.run_research.max_iterations, + }; +} + +function buildStorageStatus(config: KtxProjectConfig): StorageStatus { + return { + state: config.storage.state, + search: config.storage.search, + gitAutoCommit: config.storage.git.auto_commit, + gitAuthor: config.storage.git.author, + }; +} + +function buildWarnings( + config: KtxProjectConfig, + connections: ConnectionStatus[], + llm: LlmStatus, + embeddings: EmbeddingsStatus, +): WarningItem[] { + const warnings: WarningItem[] = []; + + for (const adapter of config.ingest.adapters) { + const requiredDrivers = ADAPTER_DRIVER_REQUIREMENT[adapter]; + if (!requiredDrivers) continue; + const hasMatching = connections.some((c) => requiredDrivers.includes(c.driver)); + if (!hasMatching) { + warnings.push({ + message: `Adapter "${adapter}" is enabled but no connection of type ${requiredDrivers.slice(0, 2).join('/')} is configured.`, + fix: 'Rerun `ktx setup` to add a connection, or remove the adapter from ingest.adapters.', + }); + } + } + + if (config.agent.run_research.enabled && llm.backend === 'none') { + warnings.push({ + message: 'Research agent is enabled but LLM is not configured.', + fix: 'Set up an LLM provider via `ktx setup` or disable agent.run_research.enabled.', + }); + } + + if (embeddings.backend === 'none' && config.ingest.adapters.includes('live-database')) { + warnings.push({ + message: 'Semantic search is off (embeddings backend = none). Lexical/dictionary lanes still work.', + }); + } + + return warnings; +} + +function buildVerdict( + llm: LlmStatus, + embeddings: EmbeddingsStatus, + connections: ConnectionStatus[], + warnings: WarningItem[], +): { verdict: ProjectVerdict; reason: string; nextActions: string[] } { + if (llm.status === 'fail') { + return { + verdict: 'blocked', + reason: 'LLM not configured — `ktx ask` will not work.', + nextActions: ['ktx setup'], + }; + } + + const reasons: string[] = []; + if (llm.status === 'warn') reasons.push('LLM credentials missing'); + if (embeddings.status === 'warn') { + if (embeddings.backend === 'deterministic' || embeddings.backend === 'none') { + reasons.push('semantic search disabled'); + } else { + reasons.push('embedding credentials missing'); + } + } + const missing = connections.filter((c) => c.status !== 'ok').length; + if (missing > 0) reasons.push(`${missing} connection${missing === 1 ? '' : 's'} need configuration`); + if (warnings.length > 0) reasons.push(`${warnings.length} config warning${warnings.length === 1 ? '' : 's'}`); + + if (reasons.length === 0) { + return { + verdict: 'ready', + reason: 'Ready.', + nextActions: ['ktx scan', 'ktx wiki', 'ktx sl ask "…"'], + }; + } + + return { + verdict: 'partial', + reason: `Partially ready — ${reasons.join('; ')}.`, + nextActions: ['ktx setup'], + }; +} + +export interface BuildProjectStatusOptions { + env?: NodeJS.ProcessEnv; +} + +export function buildProjectStatus(project: KtxLocalProject, options: BuildProjectStatusOptions = {}): ProjectStatus { + const env = options.env ?? process.env; + const config = project.config; + + const llm = buildLlmStatus(config.llm, env); + const embeddings = buildEmbeddingsStatus(config.ingest.embeddings, env); + const storage = buildStorageStatus(config); + const connections = Object.entries(config.connections).map(([name, conn]) => + buildConnectionStatus(name, conn, env), + ); + const pipeline = buildPipelineStatus(config); + const warnings = buildWarnings(config, connections, llm, embeddings); + const { verdict, reason, nextActions } = buildVerdict(llm, embeddings, connections, warnings); + + return { + projectName: config.project, + projectDir: project.projectDir, + llm, + embeddings, + storage, + connections, + pipeline, + warnings, + verdict, + verdictReason: reason, + nextActions, + promptCaching: config.llm.promptCaching + ? { + enabled: config.llm.promptCaching.enabled ?? false, + systemTtl: config.llm.promptCaching.systemTtl, + toolsTtl: config.llm.promptCaching.toolsTtl, + historyTtl: config.llm.promptCaching.historyTtl, + } + : undefined, + workUnits: { + stepBudget: config.ingest.workUnits.stepBudget, + maxConcurrency: config.ingest.workUnits.maxConcurrency, + failureMode: config.ingest.workUnits.failureMode, + }, + memoryAutoCommit: config.memory.auto_commit, + relationshipsDetail: { + acceptThreshold: config.scan.relationships.acceptThreshold, + reviewThreshold: config.scan.relationships.reviewThreshold, + maxLlmTablesPerBatch: config.scan.relationships.maxLlmTablesPerBatch, + validationConcurrency: config.scan.relationships.validationConcurrency, + }, + }; +} + +// ─── Rendering ────────────────────────────────────────────────────────────── + +const SYMBOL: Record = { ok: '✓', warn: '⚠', fail: '✗' }; + +function ansi(useColor: boolean, code: string, text: string, closer = '39'): string { + return useColor ? `\u001b[${code}m${text}\u001b[${closer}m` : text; +} + +function colorFor(level: ProjectStatusLevel): string { + return level === 'ok' ? '32' : level === 'warn' ? '33' : '31'; +} + +function abbreviateHome(filePath: string, env: NodeJS.ProcessEnv): string { + const home = env.HOME; + if (home && (filePath === home || filePath.startsWith(`${home}/`))) { + return filePath === home ? '~' : `~${filePath.slice(home.length)}`; + } + return filePath; +} + +export interface RenderProjectStatusOptions { + verbose?: boolean; + useColor?: boolean; + durationMs?: number; + toolchainChecks?: DoctorCheck[]; + env?: NodeJS.ProcessEnv; +} + +export function renderProjectStatus(status: ProjectStatus, options: RenderProjectStatusOptions = {}): string { + const verbose = options.verbose ?? false; + const useColor = options.useColor ?? false; + const env = options.env ?? process.env; + const dim = (s: string) => ansi(useColor, '2', s, '22'); + const bold = (s: string) => ansi(useColor, '1', s, '22'); + const color = (level: ProjectStatusLevel, s: string) => ansi(useColor, colorFor(level), s); + const sym = (level: ProjectStatusLevel) => color(level, SYMBOL[level]); + + const lines: string[] = []; + const dirStr = abbreviateHome(status.projectDir, env); + lines.push(`${bold('KTX status')} ${dim('·')} ${status.projectName} ${dim(`(${dirStr})`)}`); + lines.push(''); + + const labelPad = 'Connections'.length; + const label = (text: string) => text.padEnd(labelPad); + + // Core readiness rows + const llmDetail = [status.llm.backend, status.llm.model].filter(Boolean).join(` ${dim('·')} `); + lines.push(` ${label('LLM')} ${llmDetail} ${sym(status.llm.status)} ${dim(status.llm.detail)}`); + + const embedParts = [status.embeddings.backend]; + if (status.embeddings.model) embedParts.push(status.embeddings.model); + const embedDim = status.embeddings.dimensions ? `(${status.embeddings.dimensions}d)` : ''; + const embedDetail = `${embedParts.join(` ${dim('·')} `)}${embedDim ? ` ${embedDim}` : ''}`; + lines.push(` ${label('Embeddings')} ${embedDetail} ${sym(status.embeddings.status)} ${dim(status.embeddings.detail)}`); + + lines.push(` ${label('Storage')} ${dim(`${status.storage.state} (state) · ${status.storage.search} (search)`)}`); + lines.push(''); + + // Connections + if (status.connections.length === 0) { + lines.push(` ${bold('Connections')} ${dim('(none)')}`); + lines.push(` ${dim('No connections configured. Run `ktx setup` to add one.')}`); + } else { + lines.push(` ${bold('Connections')} ${dim(`(${status.connections.length})`)}`); + const nameWidth = Math.max(...status.connections.map((c) => c.name.length)); + const driverWidth = Math.max(...status.connections.map((c) => c.driver.length)); + for (const conn of status.connections) { + lines.push( + ` ${sym(conn.status)} ${conn.name.padEnd(nameWidth)} ${dim(conn.driver.padEnd(driverWidth))} ${conn.detail}`, + ); + if (conn.fix && conn.status !== 'ok') { + const indent = 6 + nameWidth + 3 + driverWidth + 3; + lines.push(`${' '.repeat(indent)}${dim(`→ ${conn.fix}`)}`); + } + } + } + lines.push(''); + + // Pipeline + lines.push(` ${bold('Pipeline')}`); + const pipelineLabelWidth = Math.max('Adapters'.length, 'Enrichment'.length, 'Research agent'.length); + const pLabel = (text: string) => text.padEnd(pipelineLabelWidth); + lines.push(` ${pLabel('Adapters')} ${status.pipeline.adapters.length > 0 ? status.pipeline.adapters.join(', ') : dim('(none)')}`); + const enrichmentDetail = [`${status.pipeline.enrichmentMode} mode`]; + if (status.pipeline.relationshipsEnabled) { + const bits = ['relationships on']; + if (status.pipeline.relationshipsLlmProposals) bits.push('LLM proposals'); + if (status.pipeline.relationshipsValidationRequired) bits.push('validation required'); + enrichmentDetail.push(bits.join(', ')); + } else { + enrichmentDetail.push('relationships off'); + } + lines.push(` ${pLabel('Enrichment')} ${enrichmentDetail.join(` ${dim('·')} `)}`); + const agentDetail = status.pipeline.agentEnabled + ? `enabled ${dim(`(${status.pipeline.agentTools.length} tool${status.pipeline.agentTools.length === 1 ? '' : 's'})`)}` + : dim('disabled'); + lines.push(` ${pLabel('Research agent')} ${agentDetail}`); + lines.push(''); + + // Warnings + if (status.warnings.length > 0) { + lines.push(` ${bold('Warnings')}`); + for (const w of status.warnings) { + lines.push(` ${color('warn', SYMBOL.warn)} ${w.message}`); + if (w.fix) lines.push(` ${dim(`→ ${w.fix}`)}`); + } + lines.push(''); + } + + // Verbose extras + if (verbose) { + if (options.toolchainChecks && options.toolchainChecks.length > 0) { + lines.push(` ${bold('Toolchain')}`); + for (const check of options.toolchainChecks) { + const lv: ProjectStatusLevel = check.status === 'pass' ? 'ok' : check.status === 'warn' ? 'warn' : 'fail'; + lines.push(` ${sym(lv)} ${check.label}: ${check.detail}`); + if (check.fix && lv !== 'ok') lines.push(` ${dim(`→ ${check.fix}`)}`); + } + lines.push(''); + } + if (status.promptCaching) { + const pc = status.promptCaching; + const bits = [`enabled=${pc.enabled}`]; + if (pc.systemTtl) bits.push(`system=${pc.systemTtl}`); + if (pc.toolsTtl) bits.push(`tools=${pc.toolsTtl}`); + if (pc.historyTtl) bits.push(`history=${pc.historyTtl}`); + lines.push(` ${bold('Prompt caching')} ${dim(bits.join(', '))}`); + } + if (status.workUnits) { + const wu = status.workUnits; + lines.push(` ${bold('Work units')} ${dim(`stepBudget=${wu.stepBudget}, maxConcurrency=${wu.maxConcurrency}, failureMode=${wu.failureMode}`)}`); + } + if (status.relationshipsDetail) { + const r = status.relationshipsDetail; + lines.push( + ` ${bold('Relationships')} ${dim(`accept=${r.acceptThreshold}, review=${r.reviewThreshold}, maxLlmTables=${r.maxLlmTablesPerBatch}, concurrency=${r.validationConcurrency}`)}`, + ); + } + lines.push( + ` ${bold('Agent')} ${dim(`max_iterations=${status.pipeline.agentMaxIterations}, tools=${status.pipeline.agentTools.join(', ') || '(none)'}`)}`, + ); + lines.push(` ${bold('Memory')} ${dim(`auto_commit=${status.memoryAutoCommit}`)}`); + lines.push( + ` ${bold('Git')} ${dim(`auto_commit=${status.storage.gitAutoCommit}, author=${status.storage.gitAuthor}`)}`, + ); + lines.push(''); + } + + // Verdict + next steps + const verdictLevel: ProjectStatusLevel = + status.verdict === 'ready' ? 'ok' : status.verdict === 'partial' ? 'warn' : 'fail'; + const duration = options.durationMs !== undefined ? ` ${dim(`(${(options.durationMs / 1000).toFixed(2)}s)`)}` : ''; + if (status.verdict === 'ready') { + const hint = ` ${dim('Try:')} ${status.nextActions.join(dim(' · '))}`; + lines.push(`${color(verdictLevel, status.verdictReason)}${hint}${duration}`); + } else { + const hint = status.nextActions.length > 0 ? ` ${dim('Next:')} ${status.nextActions.join(dim(' · '))}` : ''; + lines.push(`${color(verdictLevel, status.verdictReason)}${hint}${duration}`); + } + lines.push(''); + + return lines.join('\n'); +}