diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3da14c7b..ff0b7843 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,6 +15,46 @@ concurrency: cancel-in-progress: true jobs: + pre-commit-checks: + name: Pre-commit checks + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Setup pnpm + uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 + with: + run_install: false + + - name: Setup Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: "24" + cache: "pnpm" + cache-dependency-path: "pnpm-lock.yaml" + + - name: Install TypeScript dependencies + run: pnpm install --frozen-lockfile + + - name: Setup Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.13" + + - name: Setup uv + uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 + with: + version: "0.11.11" + enable-cache: true + cache-dependency-glob: "uv.lock" + + - name: Install Python dependencies + run: uv sync --all-packages --all-groups + + - name: Run pre-commit hooks + run: uv run pre-commit run --all-files + typescript-checks: name: TypeScript checks runs-on: ubuntu-latest @@ -23,7 +63,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup pnpm - uses: pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0 # v6.0.7 + uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 with: run_install: false @@ -51,7 +91,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup pnpm - uses: pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0 # v6.0.7 + uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 with: run_install: false @@ -79,7 +119,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup pnpm - uses: pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0 # v6.0.7 + uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 with: run_install: false @@ -107,7 +147,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup pnpm - uses: pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0 # v6.0.7 + uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 with: run_install: false @@ -156,7 +196,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup pnpm - uses: pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0 # v6.0.7 + uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 with: run_install: false diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2a8f696e..36eaf49c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,7 +24,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup pnpm - uses: pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0 # v6.0.7 + uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 with: run_install: false diff --git a/LICENSE b/LICENSE index 57bc88a1..261eeb9e 100644 --- a/LICENSE +++ b/LICENSE @@ -199,4 +199,3 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - diff --git a/docs-site/content/docs/cli-reference/ktx-connection.mdx b/docs-site/content/docs/cli-reference/ktx-connection.mdx index 0cec3eae..68b7f496 100644 --- a/docs-site/content/docs/cli-reference/ktx-connection.mdx +++ b/docs-site/content/docs/cli-reference/ktx-connection.mdx @@ -63,8 +63,7 @@ agents. "connections": [ { "id": "my-warehouse", - "driver": "postgres", - "readonly": false + "driver": "postgres" } ] } diff --git a/docs-site/content/docs/cli-reference/ktx-setup.mdx b/docs-site/content/docs/cli-reference/ktx-setup.mdx index 374fb8e7..d2348231 100644 --- a/docs-site/content/docs/cli-reference/ktx-setup.mdx +++ b/docs-site/content/docs/cli-reference/ktx-setup.mdx @@ -18,8 +18,6 @@ ktx setup [options] | Flag | Description | Default | |------|-------------|---------| | `--project-dir ` | KTX project directory | `KTX_PROJECT_DIR`, nearest `ktx.yaml`, or cwd | -| `--new` | Create a new KTX project before setup | `false` | -| `--existing` | Use an existing KTX project | `false` | | `--yes` | Accept safe defaults in non-interactive setup | `false` | | `--no-input` | Disable interactive terminal input | — | @@ -29,75 +27,11 @@ ktx setup [options] |------|-------------|---------| | `--agents` | Install agent integration only | `false` | | `--target ` | Agent target (`claude-code`, `codex`, `cursor`, `opencode`, `universal`) | — | -| `--agent-scope ` | Agent install scope (`project` or `global`) | `project` | -| `--project` | Install agent integration into the project scope | `false` | | `--global` | Install agent integration into the global target scope (Claude Code and Codex only) | `false` | -| `--skip-agents` | Leave agent integration incomplete for now | `false` | -### LLM Configuration - -| Flag | Description | Default | -|------|-------------|---------| -| `--anthropic-api-key-env ` | Environment variable containing the Anthropic API key | — | -| `--anthropic-api-key-file ` | File containing the Anthropic API key | — | -| `--anthropic-model ` | Anthropic model ID to validate and save | — | -| `--skip-llm` | Leave LLM setup incomplete for now | `false` | - -### Embedding Configuration - -| Flag | Description | Default | -|------|-------------|---------| -| `--embedding-backend ` | Embedding backend (`openai` or `sentence-transformers`) | — | -| `--embedding-api-key-env ` | Environment variable containing the embedding provider API key | — | -| `--embedding-api-key-file ` | File containing the embedding provider API key | — | -| `--skip-embeddings` | Leave embedding setup incomplete for now | `false` | - -### Database Configuration - -| Flag | Description | Default | -|------|-------------|---------| -| `--database ` | Database driver to configure; repeatable (`sqlite`, `postgres`, `mysql`, `clickhouse`, `sqlserver`, `bigquery`, `snowflake`) | — | -| `--database-connection-id ` | Existing or new connection id; repeatable | — | -| `--new-database-connection-id ` | Connection id for one new database connection | — | -| `--database-url ` | URL, `env:NAME`, or `file:/path` for one new URL-style database connection | — | -| `--database-schema ` | Database schema to include; repeatable | — | -| `--skip-databases` | Leave database setup incomplete | `false` | - -### Query history - -| Flag | Description | Default | -|------|-------------|---------| -| `--enable-query-history` | Enable query history when the selected database supports it | `false` | -| `--disable-query-history` | Disable query history for the selected database | `false` | -| `--query-history-window-days ` | Query-history lookback window in days | — | -| `--query-history-min-executions ` | Minimum executions for a query-history template | — | -| `--query-history-service-account-pattern ` | Query-history service-account regex; repeatable | — | -| `--query-history-redaction-pattern ` | Query-history SQL-literal redaction regex; repeatable | — | - -### Context Source Configuration - -| Flag | Description | Default | -|------|-------------|---------| -| `--source ` | Source connector type (`dbt`, `metricflow`, `metabase`, `looker`, `lookml`, `notion`) | — | -| `--source-connection-id ` | Connection id for source setup | — | -| `--source-path ` | Local source path for dbt, MetricFlow, or LookML | — | -| `--source-git-url ` | Git URL for dbt, MetricFlow, or LookML | — | -| `--source-branch ` | Git branch for source setup | — | -| `--source-subpath ` | Repo subpath for source setup | — | -| `--source-auth-token-ref ` | `env:` or `file:` credential ref for source repo auth | — | -| `--source-url ` | Source service URL for Metabase or Looker | — | -| `--source-api-key-ref ` | `env:` or `file:` API key ref for Metabase or Notion | — | -| `--source-client-id ` | Looker client id | — | -| `--source-client-secret-ref ` | `env:` or `file:` Looker client secret ref | — | -| `--source-warehouse-connection-id ` | Mapped warehouse connection id | — | -| `--source-project-name ` | dbt project name override | — | -| `--source-profiles-path ` | dbt profiles path | — | -| `--source-target ` | dbt target or source-specific mapping target | — | -| `--metabase-database-id ` | Metabase database id to map | — | -| `--notion-crawl-mode ` | Notion crawl mode (`all_accessible` or `selected_roots`) | — | -| `--notion-root-page-id ` | Notion root page id; repeatable | — | -| `--skip-initial-source-ingest` | Validate source setup without building source context during setup | `false` | -| `--skip-sources` | Mark optional source setup complete with no sources | `false` | +The setup wizard is the public configuration interface. It prompts for LLM +credentials, embeddings, database connections, context sources, query history, +and agent integration when those values are needed. ## Examples @@ -105,17 +39,8 @@ ktx setup [options] # Run the interactive setup wizard ktx setup -# Create a new project and run setup -ktx setup --new - -# Resume setup in an existing project -ktx setup --existing - -# Non-interactive setup with Anthropic key from environment -ktx setup --yes --anthropic-api-key-env ANTHROPIC_API_KEY - -# Set up a Postgres connection -ktx setup --database postgres --database-url "env:DATABASE_URL" +# Run setup for a specific project directory +ktx setup --project-dir ./analytics # Install agent integration for Claude Code only ktx setup --agents --target claude-code @@ -123,12 +48,6 @@ ktx setup --agents --target claude-code # Install agent integration globally for Codex ktx setup --agents --target codex --global -# Add a dbt source from a local path -ktx setup --source dbt --source-path ./my-dbt-project - -# Skip optional steps for a minimal setup -ktx setup --skip-sources --skip-agents - # Check setup readiness ktx status ``` @@ -155,5 +74,5 @@ Agent integration ready: yes (codex:project) |-------|-------|----------| | Setup resumes an unexpected project | `KTX_PROJECT_DIR` or nearest `ktx.yaml` points to another directory | Pass `--project-dir ` explicitly | | Health check for model fails | Provider key or model id is invalid | Set the correct environment variable or secret file and rerun setup | -| Setup cannot run in CI | Interactive prompts need a TTY | Use `--yes --no-input` with explicit flags for required values | +| Setup cannot run in CI | Interactive prompts need a TTY | Run setup interactively before CI, or provide a fixture `ktx.yaml` for automated tests | | Agent integration missing | Setup skipped the agents step | Run `ktx setup --agents --target ` | diff --git a/docs-site/content/docs/getting-started/quickstart.mdx b/docs-site/content/docs/getting-started/quickstart.mdx index 46a45db4..28728886 100644 --- a/docs-site/content/docs/getting-started/quickstart.mdx +++ b/docs-site/content/docs/getting-started/quickstart.mdx @@ -244,7 +244,7 @@ Agent integration ready: yes (claude-code:project) | Local embeddings hang or fail | The managed Python runtime cannot start or the local model runtime is unavailable | Install `uv`, run `ktx dev runtime status`, then run `ktx dev runtime install --feature local-embeddings --yes` and rerun setup | | Database connection test fails | Credentials, network access, warehouse, database, or schema value is wrong | Test the same URL with the database's native client, then rerun `ktx setup` and reconfigure the connection | | `KTX context built: no` in `ktx status` | Setup saved configuration but did not build context | Run `ktx setup` and choose to build context now | -| Agent integration is incomplete | Setup skipped the agents step or the target was not installed | Run `ktx setup --agents --target codex --project` using the target you need | +| Agent integration is incomplete | Setup skipped the agents step or the target was not installed | Run `ktx setup --agents --target codex` using the target you need | ## Next steps diff --git a/docs-site/content/docs/integrations/context-sources.mdx b/docs-site/content/docs/integrations/context-sources.mdx index 86c3135a..56e2f539 100644 --- a/docs-site/content/docs/integrations/context-sources.mdx +++ b/docs-site/content/docs/integrations/context-sources.mdx @@ -24,7 +24,6 @@ Agents must configure and ingest context sources in this order: | Field | Required | Description | |-------|----------|-------------| | `driver` | Yes | Source adapter: `dbt`, `metricflow`, `lookml`, `metabase`, `looker`, or `notion` | -| `readonly` | Strongly recommended | Marks the source as read-only for KTX | | `source_dir` | For local file sources | Absolute or project-relative source directory | | `repo_url` | For Git-hosted sources | Git repository URL | | `branch` | No | Git branch to read | @@ -50,7 +49,6 @@ connections: my-dbt: driver: dbt source_dir: /path/to/dbt/project - readonly: true ``` For a Git-hosted project: @@ -63,7 +61,6 @@ connections: branch: main path: analytics/dbt # For monorepos auth_token_ref: env:GITHUB_TOKEN - readonly: true ``` ### Authentication @@ -111,7 +108,6 @@ connections: branch: main path: dbt_metrics # Subdirectory for monorepos auth_token_ref: env:GITHUB_TOKEN - readonly: true ``` For a local path: @@ -158,7 +154,6 @@ connections: branch: main path: analytics # Subdirectory for monorepos auth_token_ref: env:GITHUB_TOKEN - readonly: true ``` For a local path: @@ -220,7 +215,6 @@ connections: syncEnabled: "3": true syncMode: ONLY # Only ingest mapped databases - readonly: true ``` ### Authentication @@ -277,7 +271,6 @@ connections: mappings: connectionMappings: postgres_connection: postgres-main # Looker conn → KTX conn - readonly: true ``` ### Authentication @@ -330,7 +323,6 @@ connections: crawl_mode: selected_roots root_page_ids: - "abc123def456..." - readonly: true ``` For crawling all accessible pages: @@ -341,7 +333,6 @@ connections: driver: notion auth_token_ref: env:NOTION_TOKEN crawl_mode: all_accessible - readonly: true ``` ### Authentication diff --git a/docs-site/content/docs/integrations/primary-sources.mdx b/docs-site/content/docs/integrations/primary-sources.mdx index 62b45fcf..beb9cb27 100644 --- a/docs-site/content/docs/integrations/primary-sources.mdx +++ b/docs-site/content/docs/integrations/primary-sources.mdx @@ -25,7 +25,6 @@ Agents should prefer environment or file references over literal secrets. | `url` | One of the connection methods | URL-style connectors | Database URL, `env:NAME`, or `file:/path/to/secret` | | `host`, `port`, `database`, `username`, `password` | One of the connection methods | PostgreSQL, MySQL, ClickHouse, SQL Server | Field-by-field connection values | | `schema` or `schemas` | No | schema-aware warehouses | Single schema or list of schemas to scan | -| `readonly` | Strongly recommended | all primary sources | Marks the connection as read-only in KTX config | | `context.queryHistory` | No | PostgreSQL, Snowflake, BigQuery | Enables query-history ingestion when the warehouse supports it | | `path` | Yes for path-style SQLite | SQLite | Local SQLite database path or `env:NAME` reference | @@ -39,9 +38,8 @@ The most full-featured connector. Supports schema introspection, foreign key det connections: my-postgres: driver: postgres - url: postgresql://user:password@host:5432/database + url: env:DATABASE_URL schema: public - readonly: true ``` Or with individual fields: @@ -59,7 +57,6 @@ connections: - public - analytics ssl: true - readonly: true ``` ### Authentication @@ -128,7 +125,6 @@ connections: username: KTX_SERVICE password: env:SNOWFLAKE_PASSWORD role: ANALYST - readonly: true ``` For multiple schemas: @@ -201,7 +197,6 @@ connections: credentials_json: file:~/.config/gcloud/bq-service-account.json dataset_id: analytics location: US - readonly: true ``` For multiple datasets: @@ -274,7 +269,6 @@ connections: my-clickhouse: driver: clickhouse url: http://localhost:8123/analytics - readonly: true ``` Or with individual fields: @@ -289,7 +283,6 @@ connections: username: default password: env:CH_PASSWORD ssl: false - readonly: true ``` ### Authentication @@ -332,8 +325,7 @@ Standard MySQL/MariaDB connector with full foreign key support and schema intros connections: my-mysql: driver: mysql - url: mysql://user:password@host:3306/database - readonly: true + url: env:MYSQL_DATABASE_URL ``` Or with individual fields: @@ -348,7 +340,6 @@ connections: username: ktx_reader password: env:MYSQL_PASSWORD ssl: true - readonly: true ``` ### Authentication @@ -391,8 +382,7 @@ Connects to Microsoft SQL Server and Azure SQL. Supports multi-schema scanning w connections: my-sqlserver: driver: sqlserver - url: mssql://user:password@host:1433/database?trustServerCertificate=true - readonly: true + url: env:SQLSERVER_DATABASE_URL ``` Or with individual fields: @@ -408,7 +398,6 @@ connections: password: env:MSSQL_PASSWORD schema: dbo trustServerCertificate: true - readonly: true ``` For multiple schemas: @@ -460,7 +449,6 @@ connections: my-sqlite: driver: sqlite path: ./data/warehouse.sqlite - readonly: true ``` Path supports multiple formats: diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-cross-dialect-readiness.md b/docs/superpowers/plans/2026-05-11-historic-sql-cross-dialect-readiness.md index a7a5cc6c..3fc3e496 100644 --- a/docs/superpowers/plans/2026-05-11-historic-sql-cross-dialect-readiness.md +++ b/docs/superpowers/plans/2026-05-11-historic-sql-cross-dialect-readiness.md @@ -40,37 +40,37 @@ This plan does not update `examples/postgres-historic/README.md` or `examples/po Modify: -- `packages/context/src/ingest/adapters/historic-sql/types.ts` +- `packages/context/src/ingest/adapters/historic-sql/types.ts` Adds optional probe `info` notes and lets injected historic-SQL dependencies use any reader/query client pair while preserving the existing Postgres-specific option. -- `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.ts` +- `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.ts` Moves low `pg_stat_statements.max` from `warnings` to `info`. -- `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.test.ts` Locks `track = none` as warning and low `max` as info. -- `packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.ts` +- `packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.ts` Returns `{ warnings: [], info: [] }` from `probe()`. -- `packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.test.ts` Locks the BigQuery probe return object. -- `packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.ts` +- `packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.ts` Returns `{ warnings: [], info: [] }` from `probe()`. -- `packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.test.ts` Locks the Snowflake probe return object. -- `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts` Updates test readers to return the normalized probe shape. -- `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts` Updates test readers to return the normalized probe shape. -- `packages/context/src/ingest/local-adapters.ts` +- `packages/context/src/ingest/local-adapters.ts` Accepts generic historic-SQL reader/query-client dependencies while keeping `postgresQueryClient` as the compatibility input used by current callers. -- `packages/context/src/ingest/local-adapters.test.ts` +- `packages/context/src/ingest/local-adapters.test.ts` Verifies generic reader/query-client injection and the existing Postgres compatibility path. -- `packages/cli/src/local-adapters.ts` +- `packages/cli/src/local-adapters.ts` Chooses Postgres, BigQuery, or Snowflake historic-SQL readers/query clients from the configured connection. -- `packages/cli/src/local-adapters.test.ts` +- `packages/cli/src/local-adapters.test.ts` Adds direct tests for CLI local adapter registration for Postgres, BigQuery, and Snowflake. -- `packages/cli/src/historic-sql-doctor.ts` +- `packages/cli/src/historic-sql-doctor.ts` Treats info-only Postgres probe notes as a passing doctor check, and warnings as warnings. -- `packages/cli/src/historic-sql-doctor.test.ts` +- `packages/cli/src/historic-sql-doctor.test.ts` Verifies low `pg_stat_statements.max` is pass/detail, while `track = none` remains warn. -- `packages/cli/src/doctor.test.ts` +- `packages/cli/src/doctor.test.ts` Updates the project doctor integration expectation for the new info-only behavior. ## Task 1: Normalize Historic-SQL Probe Results diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-end-to-end-retrieval-acceptance.md b/docs/superpowers/plans/2026-05-11-historic-sql-end-to-end-retrieval-acceptance.md index c9c40fd9..106131ed 100644 --- a/docs/superpowers/plans/2026-05-11-historic-sql-end-to-end-retrieval-acceptance.md +++ b/docs/superpowers/plans/2026-05-11-historic-sql-end-to-end-retrieval-acceptance.md @@ -44,7 +44,7 @@ Remaining acceptance gap this plan covers: Create: -- `packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts` Owns the end-to-end local regression for the redesigned historic-SQL pipeline. It uses the real adapter and local ingest runner, with fake deterministic reader/analysis/agent components so the test does not need a live database or LLM provider. ## Task 1: Add Real-Adapter Local Ingest Acceptance Coverage diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-foundations.md b/docs/superpowers/plans/2026-05-11-historic-sql-foundations.md index fdd97d3f..6705d56f 100644 --- a/docs/superpowers/plans/2026-05-11-historic-sql-foundations.md +++ b/docs/superpowers/plans/2026-05-11-historic-sql-foundations.md @@ -41,50 +41,50 @@ The next plan after this one should cover search enrichment from spec §6.2.3-§ Create: -- `packages/context/src/ingest/adapters/historic-sql/skill-schemas.ts` +- `packages/context/src/ingest/adapters/historic-sql/skill-schemas.ts` Owns the shared zod schemas for historic-SQL LLM outputs. -- `packages/context/src/ingest/adapters/historic-sql/skill-schemas.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/skill-schemas.test.ts` Locks schema acceptance, JSON schema generation, and future-key tolerance. -- `python/ktx-daemon/src/ktx_daemon/sql_analysis.py` +- `python/ktx-daemon/src/ktx_daemon/sql_analysis.py` Implements batch sqlglot parsing for table and clause-level column extraction. -- `python/ktx-daemon/tests/test_sql_analysis.py` +- `python/ktx-daemon/tests/test_sql_analysis.py` Tests batch parser behavior without FastAPI. Modify: -- `packages/context/src/ingest/index.ts` +- `packages/context/src/ingest/index.ts` Exports the new historic-SQL skill schemas. -- `packages/context/src/sl/types.ts` +- `packages/context/src/sl/types.ts` Adds `usage?: TableUsageOutput` to `SemanticLayerSource`. -- `packages/context/src/sl/schemas.ts` +- `packages/context/src/sl/schemas.ts` Accepts `usage` in standalone and overlay semantic-layer source validation. -- `packages/context/src/sl/semantic-layer.service.ts` +- `packages/context/src/sl/semantic-layer.service.ts` Projects manifest `usage` onto `SemanticLayerSource` and composes overlay usage intentionally. -- `packages/context/src/sl/semantic-layer.service.test.ts` +- `packages/context/src/sl/semantic-layer.service.test.ts` Tests source schema acceptance, manifest projection, and overlay composition. -- `packages/context/src/ingest/adapters/live-database/manifest.ts` +- `packages/context/src/ingest/adapters/live-database/manifest.ts` Adds `LiveDatabaseManifestTableEntry.usage`, existing-usage inputs, and `mergeUsagePreservingExternal()`. -- `packages/context/src/ingest/adapters/live-database/manifest.test.ts` +- `packages/context/src/ingest/adapters/live-database/manifest.test.ts` Tests scan-managed usage replacement while preserving external keys. -- `packages/context/src/scan/local-enrichment-artifacts.ts` +- `packages/context/src/scan/local-enrichment-artifacts.ts` Loads existing manifest usage and passes it through scan manifest rebuilds. -- `packages/context/src/scan/local-enrichment-artifacts.test.ts` +- `packages/context/src/scan/local-enrichment-artifacts.test.ts` Tests that structural scan rewrites preserve existing usage. -- `python/ktx-daemon/src/ktx_daemon/app.py` +- `python/ktx-daemon/src/ktx_daemon/app.py` Registers `/sql/analyze-batch`. -- `python/ktx-daemon/tests/test_app.py` +- `python/ktx-daemon/tests/test_app.py` Tests the FastAPI endpoint. -- `packages/context/src/sql-analysis/ports.ts` +- `packages/context/src/sql-analysis/ports.ts` Adds batch analysis types and `SqlAnalysisPort.analyzeBatch()`. -- `packages/context/src/sql-analysis/index.ts` +- `packages/context/src/sql-analysis/index.ts` Exports the new batch analysis types. -- `packages/context/src/sql-analysis/http-sql-analysis-port.ts` +- `packages/context/src/sql-analysis/http-sql-analysis-port.ts` Maps `/sql/analyze-batch` request and response payloads. -- `packages/context/src/sql-analysis/http-sql-analysis-port.test.ts` +- `packages/context/src/sql-analysis/http-sql-analysis-port.test.ts` Tests HTTP mapping and malformed response rejection. -- `packages/cli/src/managed-python-http.test.ts` +- `packages/cli/src/managed-python-http.test.ts` Verifies the managed daemon wrapper routes `analyzeBatch()`. -- Existing test files with `SqlAnalysisPort` object literals +- Existing test files with `SqlAnalysisPort` object literals Add a no-op `analyzeBatch: async () => new Map()` while legacy paths still use `analyzeForFingerprint()`. ## Task 1: Add Historic SQL Skill Schemas diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-pattern-shard-smoke-docs.md b/docs/superpowers/plans/2026-05-11-historic-sql-pattern-shard-smoke-docs.md index 9e386a16..b5382ff4 100644 --- a/docs/superpowers/plans/2026-05-11-historic-sql-pattern-shard-smoke-docs.md +++ b/docs/superpowers/plans/2026-05-11-historic-sql-pattern-shard-smoke-docs.md @@ -39,13 +39,13 @@ Remaining gap this plan fixes: ## File Structure -- Modify `scripts/examples-docs.test.mjs` +- Modify `scripts/examples-docs.test.mjs` Pins docs and smoke script to the sharded pattern WorkUnit contract. -- Modify `examples/postgres-historic/scripts/smoke.sh` +- Modify `examples/postgres-historic/scripts/smoke.sh` Validates `patterns-input/part-*.json` shard files and `historic-sql-patterns-part-*` stage-only WorkUnits. -- Modify `examples/postgres-historic/README.md` +- Modify `examples/postgres-historic/README.md` Documents `patterns-input.json` as the full audit artifact and `patterns-input/part-*.json` as bounded pattern WorkUnit input. -- Modify `examples/README.md` +- Modify `examples/README.md` Updates the short example catalog entry with the same audit-vs-shard wording. ### Task 1: Pin Example Tests To Pattern Shards diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-pattern-workunit-sharding.md b/docs/superpowers/plans/2026-05-11-historic-sql-pattern-workunit-sharding.md index ee7604a7..c67f6d78 100644 --- a/docs/superpowers/plans/2026-05-11-historic-sql-pattern-workunit-sharding.md +++ b/docs/superpowers/plans/2026-05-11-historic-sql-pattern-workunit-sharding.md @@ -30,23 +30,23 @@ No existing spec-derived plan is currently unimplemented in this worktree. This ## File Structure -- Create `packages/context/src/ingest/adapters/historic-sql/pattern-inputs.ts` +- Create `packages/context/src/ingest/adapters/historic-sql/pattern-inputs.ts` Owns deterministic pattern audit ordering, cross-table candidate filtering, byte-bounded shard creation, shard path constants, and shard path detection. -- Create `packages/context/src/ingest/adapters/historic-sql/pattern-inputs.test.ts` +- Create `packages/context/src/ingest/adapters/historic-sql/pattern-inputs.test.ts` Covers deterministic shard ordering, single-table exclusion from WorkUnit shards, byte limits, and oversize-template manifest warnings. -- Modify `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts` +- Modify `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts` Writes full `patterns-input.json` plus bounded `patterns-input/part-0001.json` shard files, and appends shard warnings to `manifest.json`. -- Modify `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts` +- Modify `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts` Adds a regression for audit file preservation and sharded WorkUnit input creation. -- Modify `packages/context/src/ingest/adapters/historic-sql/chunk-unified.ts` +- Modify `packages/context/src/ingest/adapters/historic-sql/chunk-unified.ts` Emits one patterns WorkUnit per changed shard path, treats root `patterns-input.json` as audit-only, and includes shard paths in the scope descriptor and eviction calculation. -- Modify `packages/context/src/ingest/adapters/historic-sql/chunk-unified.test.ts` +- Modify `packages/context/src/ingest/adapters/historic-sql/chunk-unified.test.ts` Updates root-file expectations and adds multi-shard diff behavior. -- Modify `packages/context/skills/historic_sql_patterns/SKILL.md` +- Modify `packages/context/skills/historic_sql_patterns/SKILL.md` Tells the skill to read the exact pattern shard in `rawFiles` and emit evidence with that shard as `rawPath`. -- Modify `packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts` +- Modify `packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts` Updates the fake agent to emit pattern evidence for `historic-sql-patterns-part-0001`. -- Modify `packages/context/src/ingest/ingest-runtime-assets.test.ts` +- Modify `packages/context/src/ingest/ingest-runtime-assets.test.ts` Keeps packaged skill assertions aligned with sharded pattern file guidance. ## Task 1: Add Pattern Input Sharding Helper diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-redaction-hardening.md b/docs/superpowers/plans/2026-05-11-historic-sql-redaction-hardening.md index 1adcdfd3..e59e164b 100644 --- a/docs/superpowers/plans/2026-05-11-historic-sql-redaction-hardening.md +++ b/docs/superpowers/plans/2026-05-11-historic-sql-redaction-hardening.md @@ -55,16 +55,16 @@ Remaining spec gap this plan covers: Create: -- `packages/context/src/ingest/adapters/historic-sql/redaction.ts` +- `packages/context/src/ingest/adapters/historic-sql/redaction.ts` Owns compilation and application of historic-SQL SQL-text redaction patterns. Supports JavaScript regex strings and the documented `(?i)` case-insensitive prefix used by setup tests/docs. -- `packages/context/src/ingest/adapters/historic-sql/redaction.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/redaction.test.ts` Tests raw regex replacement, `(?i)` compatibility, empty config behavior, and invalid-pattern diagnostics. Modify: -- `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts` +- `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts` Compiles `config.redactionPatterns` once per fetch. Keeps original SQL for filtering and `SqlAnalysisPort.analyzeBatch()`, then stores redacted SQL in `ParsedTemplate.template.canonicalSql` before `toStagedTable()` and `toPatternsInput()` serialize files. -- `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts` Adds a regression proving raw secrets are absent from staged artifacts while `analyzeBatch()` still receives the original SQL. ## Task 1: Add Historic SQL Redaction Helper @@ -89,7 +89,7 @@ describe('historic-SQL redaction', () => { ]); const sql = - "select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; + "select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; // pragma: allowlist secret expect(redactHistoricSqlText(sql, redactors)).toBe( "select * from public.api_events where api_key = '[REDACTED]' and note = '[REDACTED]'", @@ -202,7 +202,7 @@ Append this test inside the existing `describe('stageHistoricSqlAggregatedSnapsh it('redacts configured SQL substrings in staged artifacts while analyzing original SQL', async () => { const stagedDir = await tempDir(); const originalSql = - "select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; + "select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; // pragma: allowlist secret const reader: HistoricSqlReader = { async probe() { return { warnings: [], info: [] }; diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-search-enrichment.md b/docs/superpowers/plans/2026-05-11-historic-sql-search-enrichment.md index ee960bb8..cafc234b 100644 --- a/docs/superpowers/plans/2026-05-11-historic-sql-search-enrichment.md +++ b/docs/superpowers/plans/2026-05-11-historic-sql-search-enrichment.md @@ -37,27 +37,27 @@ This plan does not rewrite the historic-SQL adapter, readers, skills, projection Modify: -- `packages/context/src/sl/sl-search.service.ts` +- `packages/context/src/sl/sl-search.service.ts` Adds usage narrative, frequency, filters, group-bys, joins, and stale marker to the canonical SL search text. Preserves snippets returned by repository search for direct `SlSearchService.search()` callers. -- `packages/context/src/sl/sl-search.service.test.ts` +- `packages/context/src/sl/sl-search.service.test.ts` Tests usage search-text content and direct service snippet pass-through. -- `packages/context/src/sl/ports.ts` +- `packages/context/src/sl/ports.ts` Extends `SlSourcesIndexPort.search()` rows with optional `snippet`. -- `packages/context/src/sl/sqlite-sl-sources-index.ts` +- `packages/context/src/sl/sqlite-sl-sources-index.ts` Adds FTS5 `snippet()` selection to lexical candidate search and direct index search. -- `packages/context/src/sl/sqlite-sl-sources-index.test.ts` +- `packages/context/src/sl/sqlite-sl-sources-index.test.ts` Locks snippet behavior for both direct search and lexical lane candidates. -- `packages/context/src/sl/local-sl.ts` +- `packages/context/src/sl/local-sl.ts` Adds `frequencyTier` and `snippet` to query-mode `LocalSlSourceSearchResult`; collects snippets from the lexical lane and hydrates frequency from `SemanticLayerSource.usage`. -- `packages/context/src/sl/local-sl.test.ts` +- `packages/context/src/sl/local-sl.test.ts` Tests that usage-only terms can find a source and that results include `frequencyTier` and FTS snippet. -- `packages/context/src/sl/pglite-sl-search-prototype.ts` +- `packages/context/src/sl/pglite-sl-search-prototype.ts` Propagates `frequencyTier` for the prototype backend so the shared result type stays truthful. -- `packages/context/src/mcp/types.ts` +- `packages/context/src/mcp/types.ts` Adds `frequencyTier` and `snippet` to `KtxSemanticLayerSourceSummary`. -- `packages/context/src/mcp/local-project-ports.ts` +- `packages/context/src/mcp/local-project-ports.ts` Includes `frequencyTier` and `snippet` in `semanticLayer.listSources()` output. -- `packages/context/src/mcp/local-project-ports.test.ts` +- `packages/context/src/mcp/local-project-ports.test.ts` Tests the agent/MCP-facing list response. ## Task 1: Index Historic SQL Usage In SL Search Text diff --git a/docs/superpowers/plans/2026-05-11-historic-sql-skills-projection-cutover.md b/docs/superpowers/plans/2026-05-11-historic-sql-skills-projection-cutover.md index a892542e..a7494e2d 100644 --- a/docs/superpowers/plans/2026-05-11-historic-sql-skills-projection-cutover.md +++ b/docs/superpowers/plans/2026-05-11-historic-sql-skills-projection-cutover.md @@ -52,58 +52,58 @@ Still not implemented: Create: -- `packages/context/src/ingest/adapters/historic-sql/evidence.ts` +- `packages/context/src/ingest/adapters/historic-sql/evidence.ts` Owns typed evidence envelopes, ignored evidence path helpers, and load/write helpers for table usage and pattern evidence. -- `packages/context/src/ingest/adapters/historic-sql/evidence.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/evidence.test.ts` Tests evidence schema validation, path normalization, and loader rejection of malformed evidence. -- `packages/context/src/ingest/adapters/historic-sql/evidence-tool.ts` +- `packages/context/src/ingest/adapters/historic-sql/evidence-tool.ts` Adds `emit_historic_sql_evidence`, the only write tool the two new historic-SQL skills use. -- `packages/context/src/ingest/adapters/historic-sql/evidence-tool.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/evidence-tool.test.ts` Tests the tool writes ignored run-local JSON with `skipLock: true` and rejects non-historic ingest sessions. -- `packages/context/src/ingest/adapters/historic-sql/projection.ts` +- `packages/context/src/ingest/adapters/historic-sql/projection.ts` Projects table usage evidence into manifest shards, writes pattern wiki pages, marks stale usage/pages, and deletes legacy query pages. -- `packages/context/src/ingest/adapters/historic-sql/projection.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/projection.test.ts` Tests `_schema` merge, stale usage, pattern slug reuse, stale page tagging, archive movement, and legacy page cleanup. -- `packages/context/src/ingest/adapters/historic-sql/post-processor.ts` +- `packages/context/src/ingest/adapters/historic-sql/post-processor.ts` Implements `IngestBundlePostProcessorPort` for the deterministic projection phase. -- `packages/context/src/ingest/adapters/historic-sql/post-processor.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/post-processor.test.ts` Tests post-processor path resolution from `workdir`, `connectionId`, `sourceKey`, and `syncId`. -- `packages/context/skills/historic_sql_table_digest/SKILL.md` +- `packages/context/skills/historic_sql_table_digest/SKILL.md` Skill for one changed `tables/*.json` WorkUnit; emits one table usage evidence object. -- `packages/context/skills/historic_sql_patterns/SKILL.md` +- `packages/context/skills/historic_sql_patterns/SKILL.md` Skill for `patterns-input.json`; emits one pattern evidence object per recurring cross-table intent. Modify: -- `packages/context/src/ingest/adapters/historic-sql/types.ts` +- `packages/context/src/ingest/adapters/historic-sql/types.ts` Keep only unified config/staged schemas and reader contracts; extend config preprocessing for existing `serviceAccountUserPatterns` and `minCalls` aliases. -- `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts` +- `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts` Add `staleArchiveAfterDays` to `manifest.json` so projection can archive stale pattern pages deterministically. -- `packages/context/src/ingest/adapters/historic-sql/chunk-unified.ts` +- `packages/context/src/ingest/adapters/historic-sql/chunk-unified.ts` Keep the same WorkUnits, but mention `emit_historic_sql_evidence` in `notes`. -- `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.ts` +- `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.ts` Switch production fetch/chunk/scope to the unified hot path, replace skills, remove legacy triage support, and run legacy PGSS baseline cache cleanup. -- `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts` +- `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts` Rewrite around unified staging and new skills. -- `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.ts` +- `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.ts` Inline the PGSS probe logic so `postgres-pgss-query-history-reader.ts` can be deleted. -- `packages/context/src/ingest/local-adapters.ts` +- `packages/context/src/ingest/local-adapters.ts` Use `PostgresPgssReader` for local Postgres historic SQL and return unified pull config. -- `packages/context/src/ingest/local-bundle-runtime.ts` +- `packages/context/src/ingest/local-bundle-runtime.ts` Add the source-specific evidence tool to historic-SQL WorkUnits and register the historic-SQL post-processor. -- `packages/context/src/ingest/ingest-runtime-assets.test.ts` +- `packages/context/src/ingest/ingest-runtime-assets.test.ts` Replace old skill asset assertions with the two new skills. -- `packages/context/src/memory/memory-runtime-assets.test.ts` +- `packages/context/src/memory/memory-runtime-assets.test.ts` Replace old historic-SQL skill heading with the two new skill headings. -- `packages/context/src/package-exports.test.ts` +- `packages/context/src/package-exports.test.ts` Remove legacy export assertions and add evidence/projection export assertions. -- `packages/context/src/ingest/index.ts` +- `packages/context/src/ingest/index.ts` Export new evidence/projection/post-processor helpers and remove legacy historic-SQL exports. -- `packages/cli/src/setup-databases.ts` and `packages/cli/src/historic-sql-doctor.ts` +- `packages/cli/src/setup-databases.ts` and `packages/cli/src/historic-sql-doctor.ts` Import `PostgresPgssReader` instead of `PostgresPgssQueryHistoryReader`. -- `packages/cli/src/commands/setup-commands.ts`, `packages/cli/src/index.test.ts`, `packages/cli/src/setup-databases.test.ts` +- `packages/cli/src/commands/setup-commands.ts`, `packages/cli/src/index.test.ts`, `packages/cli/src/setup-databases.test.ts` Rename generated config to `minExecutions` while accepting the old `--historic-sql-min-calls` flag for one release. -- `packages/context/prompts/skills/page_triage_classifier.md`, `packages/context/src/ingest/page-triage/page-triage.service.test.ts`, `packages/context/src/ingest/ingest-prompts.test.ts` +- `packages/context/prompts/skills/page_triage_classifier.md`, `packages/context/src/ingest/page-triage/page-triage.service.test.ts`, `packages/context/src/ingest/ingest-prompts.test.ts` Remove historic-SQL template triage examples because the new adapter no longer uses page triage. Delete: diff --git a/examples/local-warehouse/ktx.yaml b/examples/local-warehouse/ktx.yaml index a6054b4a..a967e31c 100644 --- a/examples/local-warehouse/ktx.yaml +++ b/examples/local-warehouse/ktx.yaml @@ -2,7 +2,6 @@ project: local-warehouse connections: warehouse: driver: postgres - readonly: true storage: state: sqlite search: sqlite-fts5 diff --git a/examples/orbit-relationship-verification/ktx.yaml b/examples/orbit-relationship-verification/ktx.yaml index 92a871e6..bcfad298 100644 --- a/examples/orbit-relationship-verification/ktx.yaml +++ b/examples/orbit-relationship-verification/ktx.yaml @@ -3,7 +3,6 @@ connections: orbit: driver: sqlite path: ../../packages/context/test/fixtures/relationship-benchmarks/orbit_style_product_no_declared_constraints/data.sqlite - readonly: true storage: state: sqlite search: sqlite-fts5 diff --git a/packages/cli/src/cli-program.ts b/packages/cli/src/cli-program.ts index 82dafbd3..93f31ae9 100644 --- a/packages/cli/src/cli-program.ts +++ b/packages/cli/src/cli-program.ts @@ -178,6 +178,10 @@ function shouldSuppressProjectDirLine(path: string[], options: Record { + const { runKtxTextIngest } = await import('./text-ingest.js'); + return await (ingestDeps.textIngest ?? runKtxTextIngest)(textIngestArgs, ingestIo); + }, + }); registerWikiCommands(program, context); registerSlCommands(program, context); registerStatusCommands(program, context); diff --git a/packages/cli/src/cli-runtime.ts b/packages/cli/src/cli-runtime.ts index b3ab3ed0..a2147904 100644 --- a/packages/cli/src/cli-runtime.ts +++ b/packages/cli/src/cli-runtime.ts @@ -8,6 +8,7 @@ import type { KtxRuntimeArgs } from './runtime.js'; import type { KtxSetupArgs } from './setup.js'; import type { KtxSlArgs } from './sl.js'; import { profileMark, profileSpan } from './startup-profile.js'; +import type { KtxTextIngestArgs } from './text-ingest.js'; profileMark('module:cli-runtime'); @@ -29,6 +30,7 @@ export interface KtxCliDeps { connection?: (args: KtxConnectionArgs, io: KtxCliIo) => Promise; doctor?: (args: KtxDoctorArgs, io: KtxCliIo) => Promise; publicIngest?: (args: KtxPublicIngestArgs, io: KtxCliIo) => Promise; + textIngest?: (args: KtxTextIngestArgs, io: KtxCliIo) => Promise; runtime?: (args: KtxRuntimeArgs, io: KtxCliIo) => Promise; knowledge?: (args: KtxKnowledgeArgs, io: KtxCliIo) => Promise; sl?: (args: KtxSlArgs, io: KtxCliIo) => Promise; diff --git a/packages/cli/src/commands/ingest-commands.ts b/packages/cli/src/commands/ingest-commands.ts index dcd371ea..01e262c7 100644 --- a/packages/cli/src/commands/ingest-commands.ts +++ b/packages/cli/src/commands/ingest-commands.ts @@ -1,16 +1,27 @@ import { type Command, Option } from '@commander-js/extra-typings'; import { + collectOption, type KtxCliCommandContext, parsePositiveIntegerOption, resolveCommandProjectDir, } from '../cli-program.js'; +import type { KtxCliDeps, KtxCliIo } from '../index.js'; import { runtimeInstallPolicyFromFlags } from '../managed-python-command.js'; import type { KtxPublicIngestArgs } from '../public-ingest.js'; import { profileMark } from '../startup-profile.js'; +import type { KtxTextIngestArgs } from '../text-ingest.js'; profileMark('module:commands/ingest-commands'); -export function registerIngestCommands(program: Command, context: KtxCliCommandContext): void { +interface IngestCommandOptions { + runTextIngest: (args: KtxTextIngestArgs, io: KtxCliIo, deps: KtxCliDeps) => Promise; +} + +export function registerIngestCommands( + program: Command, + context: KtxCliCommandContext, + commandOptions: IngestCommandOptions, +): void { const ingest = program .command('ingest') .description('Build or inspect KTX context') @@ -51,4 +62,32 @@ export function registerIngestCommands(program: Command, context: KtxCliCommandC ingest.hook('preAction', (_thisCommand, actionCommand) => { context.writeDebug?.('ingest', actionCommand); }); + + ingest + .command('text') + .description('Ingest free-form text artifacts into KTX memory') + .argument('[files...]', 'Files to ingest; use - to read one item from stdin') + .option('--text ', 'Text content to ingest; repeat for a batch', collectOption, []) + .option('--connection-id ', 'Optional KTX connection id for semantic-layer capture') + .option('--user-id ', 'Memory user id for capture attribution', 'local-cli') + .option('--json', 'Print JSON output') + .option('--fail-fast', 'Stop after the first failed text item', false) + .action(async (files: string[], options, command) => { + const parentOptions = command.parent?.opts() as { json?: boolean } | undefined; + context.setExitCode( + await commandOptions.runTextIngest( + { + projectDir: resolveCommandProjectDir(command), + texts: options.text, + files, + ...(options.connectionId ? { connectionId: options.connectionId } : {}), + userId: options.userId, + json: options.json === true || parentOptions?.json === true, + failFast: options.failFast === true, + }, + context.io, + context.deps, + ), + ); + }); } diff --git a/packages/cli/src/commands/setup-commands.ts b/packages/cli/src/commands/setup-commands.ts index 0ea7a189..4f6f0c32 100644 --- a/packages/cli/src/commands/setup-commands.ts +++ b/packages/cli/src/commands/setup-commands.ts @@ -64,13 +64,6 @@ function sourceType(value: string): KtxSetupSourceType { throw new InvalidArgumentError(`invalid choice '${value}'`); } -function agentScope(value: string): 'project' | 'global' { - if (value === 'project' || value === 'global') { - return value; - } - throw new InvalidArgumentError(`invalid choice '${value}'`); -} - function positiveNumber(value: string): number { const parsed = Number.parseInt(value, 10); if (!Number.isInteger(parsed) || parsed <= 0) { @@ -97,7 +90,6 @@ function shouldShowSetupEntryMenu( agents?: boolean; target?: string; global?: boolean; - project?: boolean; skipAgents?: boolean; yes?: boolean; input?: boolean; @@ -142,7 +134,6 @@ function shouldShowSetupEntryMenu( metabaseDatabaseId?: number; notionCrawlMode?: string; notionRootPageId?: string[]; - skipInitialSourceIngest?: boolean; skipSources?: boolean; }, command: Command, @@ -172,7 +163,6 @@ function shouldShowSetupEntryMenu( 'agents', 'target', 'global', - 'project', 'skipAgents', 'yes', 'input', @@ -211,7 +201,6 @@ function shouldShowSetupEntryMenu( 'sourceTarget', 'metabaseDatabaseId', 'notionCrawlMode', - 'skipInitialSourceIngest', 'skipSources', ].some((optionName) => optionWasSpecified(command, optionName)); } @@ -220,9 +209,9 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo const setup = program .command('setup') .description('Set up or resume a local KTX project') - .option('--project-dir ', 'KTX project directory') - .option('--new', 'Create a new KTX project before setup', false) - .option('--existing', 'Use an existing KTX project', false) + .addOption(new Option('--project-dir ', 'KTX project directory').hideHelp()) + .addOption(new Option('--new', 'Create a new KTX project before setup').hideHelp().default(false)) + .addOption(new Option('--existing', 'Use an existing KTX project').hideHelp().default(false)) .option('--agents', 'Install agent integration only', false) .addOption( new Option('--target ', 'Agent target').choices([ @@ -233,94 +222,128 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo 'universal', ]), ) - .addOption(new Option('--agent-scope ', 'Agent install scope').argParser(agentScope).default('project')) - .option('--project', 'Install agent integration into the project scope', false) .option('--global', 'Install agent integration into the global target scope', false) - .option('--skip-agents', 'Leave agent integration incomplete for now', false) + .addOption(new Option('--skip-agents', 'Leave agent integration incomplete for now').hideHelp().default(false)) .option('--yes', 'Accept safe defaults in non-interactive setup', false) .option('--no-input', 'Disable interactive terminal input') - .addOption(new Option('--llm-backend ', 'LLM backend').argParser(llmBackend)) - .option('--anthropic-api-key-env ', 'Environment variable containing the Anthropic API key') - .option('--anthropic-api-key-file ', 'File containing the Anthropic API key') - .option('--anthropic-model ', 'Anthropic model ID to validate and save') - .option('--vertex-project ', 'Google Vertex AI project ID, env:NAME, or file:/path') - .option('--vertex-location ', 'Google Vertex AI location, env:NAME, or file:/path') - .addOption(new Option('--skip-llm', 'Leave LLM setup incomplete for now').hideHelp().default(false)) - .addOption(new Option('--embedding-backend ', 'Embedding backend').argParser(embeddingBackend)) - .option('--embedding-api-key-env ', 'Environment variable containing the embedding provider API key') - .option('--embedding-api-key-file ', 'File containing the embedding provider API key') - .addOption(new Option('--skip-embeddings', 'Leave embedding setup incomplete for now').hideHelp().default(false)) - .option( - '--database ', - 'Database driver to configure; repeatable', - (value, previous: KtxSetupDatabaseDriver[]) => { - return [...previous, databaseDriver(value)]; - }, - [] as KtxSetupDatabaseDriver[], - ) - .option( - '--database-connection-id ', - 'Existing selected connection id or new connection id', - (value, previous: string[]) => [...previous, value], - [], - ) - .option('--new-database-connection-id ', 'Connection id for one new database connection', (value) => { - if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(value)) { - throw new InvalidArgumentError(`Unsafe connection id: ${value}`); - } - return value; - }) - .option('--database-url ', 'URL, env:NAME, or file:/path for one new URL-style database connection') - .option( - '--database-schema ', - 'Database schema to include; repeatable', - (value, previous: string[]) => [...previous, value], - [], - ) - .option('--enable-query-history', 'Enable query history when the selected database supports it', false) - .option('--disable-query-history', 'Disable query history for the selected database', false) - .option('--query-history-window-days ', 'Query-history lookback window', positiveInteger) - .option('--query-history-min-executions ', 'Minimum executions for a query-history template', positiveInteger) - .option( - '--query-history-service-account-pattern ', - 'Query-history service-account regex; repeatable', - (value, previous: string[]) => [...previous, value], - [], - ) - .option( - '--query-history-redaction-pattern ', - 'Query-history SQL-literal redaction regex; repeatable', - (value, previous: string[]) => [...previous, value], - [], - ) - .option('--skip-databases', 'Leave database setup incomplete; KTX cannot work until a database is added', false) - .addOption(new Option('--source ', 'Source connector type').argParser(sourceType)) - .option('--source-connection-id ', 'Connection id for source setup') - .option('--source-path ', 'Local source path for dbt, MetricFlow, or LookML') - .option('--source-git-url ', 'Git URL for dbt, MetricFlow, or LookML') - .option('--source-branch ', 'Git branch for source setup') - .option('--source-subpath ', 'Repo subpath for source setup') - .option('--source-auth-token-ref ', 'env: or file: credential ref for source repo auth') - .option('--source-url ', 'Source service URL for Metabase or Looker') - .option('--source-api-key-ref ', 'env: or file: API key ref for Metabase or Notion') - .option('--source-client-id ', 'Looker client id') - .option('--source-client-secret-ref ', 'env: or file: Looker client secret ref') - .option('--source-warehouse-connection-id ', 'Mapped warehouse connection id') - .option('--source-project-name ', 'dbt project name override') - .option('--source-profiles-path ', 'dbt profiles path') - .option('--source-target ', 'dbt target or source-specific mapping target') - .option('--metabase-database-id ', 'Metabase database id to map', positiveNumber) + .addOption(new Option('--llm-backend ', 'LLM backend').argParser(llmBackend).hideHelp()) .addOption( - new Option('--notion-crawl-mode ', 'Notion crawl mode').choices(['all_accessible', 'selected_roots']), + new Option('--anthropic-api-key-env ', 'Environment variable containing the Anthropic API key').hideHelp(), ) - .option( - '--notion-root-page-id ', - 'Notion root page id; repeatable', - (value, previous: string[]) => [...previous, value], - [], + .addOption( + new Option('--anthropic-api-key-file ', 'File containing the Anthropic API key').hideHelp(), ) - .option('--skip-initial-source-ingest', 'Validate source setup without building source context during setup', false) - .option('--skip-sources', 'Mark optional source setup complete with no sources', false) + .addOption(new Option('--anthropic-model ', 'Anthropic model ID to validate and save').hideHelp()) + .addOption(new Option('--vertex-project ', 'Google Vertex AI project ID, env:NAME, or file:/path').hideHelp()) + .addOption(new Option('--vertex-location ', 'Google Vertex AI location, env:NAME, or file:/path').hideHelp()) + .addOption(new Option('--skip-llm', 'Leave LLM setup incomplete for now').hideHelp().default(false)) + .addOption(new Option('--embedding-backend ', 'Embedding backend').argParser(embeddingBackend).hideHelp()) + .addOption( + new Option( + '--embedding-api-key-env ', + 'Environment variable containing the embedding provider API key', + ).hideHelp(), + ) + .addOption( + new Option('--embedding-api-key-file ', 'File containing the embedding provider API key').hideHelp(), + ) + .addOption(new Option('--skip-embeddings', 'Leave embedding setup incomplete for now').hideHelp().default(false)) + .addOption( + new Option('--database ', 'Database driver to configure; repeatable') + .argParser((value, previous: KtxSetupDatabaseDriver[]) => { + return [...previous, databaseDriver(value)]; + }) + .default([] as KtxSetupDatabaseDriver[]) + .hideHelp(), + ) + .addOption( + new Option('--database-connection-id ', 'Existing selected connection id or new connection id') + .argParser((value, previous: string[]) => [...previous, value]) + .default([] as string[]) + .hideHelp(), + ) + .addOption( + new Option('--new-database-connection-id ', 'Connection id for one new database connection') + .argParser((value) => { + if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(value)) { + throw new InvalidArgumentError(`Unsafe connection id: ${value}`); + } + return value; + }) + .hideHelp(), + ) + .addOption( + new Option('--database-url ', 'URL, env:NAME, or file:/path for one new URL-style database connection').hideHelp(), + ) + .addOption( + new Option('--database-schema ', 'Database schema to include; repeatable') + .argParser((value, previous: string[]) => [...previous, value]) + .default([] as string[]) + .hideHelp(), + ) + .addOption( + new Option('--enable-query-history', 'Enable query history when the selected database supports it') + .hideHelp() + .default(false), + ) + .addOption( + new Option('--disable-query-history', 'Disable query history for the selected database').hideHelp().default(false), + ) + .addOption( + new Option('--query-history-window-days ', 'Query-history lookback window') + .argParser(positiveInteger) + .hideHelp(), + ) + .addOption( + new Option('--query-history-min-executions ', 'Minimum executions for a query-history template') + .argParser(positiveInteger) + .hideHelp(), + ) + .addOption( + new Option('--query-history-service-account-pattern ', 'Query-history service-account regex; repeatable') + .argParser((value, previous: string[]) => [...previous, value]) + .default([] as string[]) + .hideHelp(), + ) + .addOption( + new Option('--query-history-redaction-pattern ', 'Query-history SQL-literal redaction regex; repeatable') + .argParser((value, previous: string[]) => [...previous, value]) + .default([] as string[]) + .hideHelp(), + ) + .addOption( + new Option('--skip-databases', 'Leave database setup incomplete; KTX cannot work until a database is added') + .hideHelp() + .default(false), + ) + .addOption(new Option('--source ', 'Source connector type').argParser(sourceType).hideHelp()) + .addOption(new Option('--source-connection-id ', 'Connection id for source setup').hideHelp()) + .addOption(new Option('--source-path ', 'Local source path for dbt, MetricFlow, or LookML').hideHelp()) + .addOption(new Option('--source-git-url ', 'Git URL for dbt, MetricFlow, or LookML').hideHelp()) + .addOption(new Option('--source-branch ', 'Git branch for source setup').hideHelp()) + .addOption(new Option('--source-subpath ', 'Repo subpath for source setup').hideHelp()) + .addOption(new Option('--source-auth-token-ref ', 'env: or file: credential ref for source repo auth').hideHelp()) + .addOption(new Option('--source-url ', 'Source service URL for Metabase or Looker').hideHelp()) + .addOption(new Option('--source-api-key-ref ', 'env: or file: API key ref for Metabase or Notion').hideHelp()) + .addOption(new Option('--source-client-id ', 'Looker client id').hideHelp()) + .addOption(new Option('--source-client-secret-ref ', 'env: or file: Looker client secret ref').hideHelp()) + .addOption(new Option('--source-warehouse-connection-id ', 'Mapped warehouse connection id').hideHelp()) + .addOption(new Option('--source-project-name ', 'dbt project name override').hideHelp()) + .addOption(new Option('--source-profiles-path ', 'dbt profiles path').hideHelp()) + .addOption(new Option('--source-target ', 'dbt target or source-specific mapping target').hideHelp()) + .addOption(new Option('--metabase-database-id ', 'Metabase database id to map').argParser(positiveNumber).hideHelp()) + .addOption( + new Option('--notion-crawl-mode ', 'Notion crawl mode') + .choices(['all_accessible', 'selected_roots']) + .hideHelp(), + ) + .addOption( + new Option('--notion-root-page-id ', 'Notion root page id; repeatable') + .argParser((value, previous: string[]) => [...previous, value]) + .default([] as string[]) + .hideHelp(), + ) + .addOption(new Option('--skip-sources', 'Mark optional source setup complete with no sources').hideHelp().default(false)) .showHelpAfterError(); setup.hook('preAction', (_thisCommand, actionCommand) => { @@ -371,7 +394,7 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo } const mode = options.new ? 'new' : options.existing ? 'existing' : 'auto'; - const resolvedAgentScope = options.global ? 'global' : options.agentScope; + const resolvedAgentScope = options.global ? 'global' : 'project'; await runSetupArgs(context, { command: 'run', projectDir: resolveCommandProjectDir(command), diff --git a/packages/cli/src/connection.test.ts b/packages/cli/src/connection.test.ts index 57ed8742..0d592b00 100644 --- a/packages/cli/src/connection.test.ts +++ b/packages/cli/src/connection.test.ts @@ -94,7 +94,7 @@ describe('runKtxConnection', () => { const projectDir = join(tempDir, 'project'); await initKtxProject({ projectDir, projectName: 'warehouse' }); await writeConnections(projectDir, { - warehouse: { driver: 'postgres', url: 'env:DATABASE_URL', readonly: true }, + warehouse: { driver: 'postgres', url: 'env:DATABASE_URL' }, docs: { driver: 'notion', auth_token_ref: 'env:NOTION_TOKEN', crawl_mode: 'all_accessible' }, }); const io = makeIo(); @@ -123,7 +123,7 @@ describe('runKtxConnection', () => { const projectDir = join(tempDir, 'project'); await initKtxProject({ projectDir, projectName: 'warehouse' }); await writeConnections(projectDir, { - warehouse: { driver: 'sqlite', readonly: true }, + warehouse: { driver: 'sqlite' }, }); const { connector, introspect, cleanup } = nativeConnector('sqlite', ['customers', 'orders']); const createScanConnector = vi.fn(async () => connector); @@ -159,7 +159,7 @@ describe('runKtxConnection', () => { prod_metabase: { driver: 'metabase', api_url: 'http://metabase.example.test', - api_key: 'mb_test', + api_key: 'mb_test', // pragma: allowlist secret }, }); const testConnection = vi.fn(async () => ({ success: true as const })); @@ -202,7 +202,7 @@ describe('runKtxConnection', () => { const projectDir = join(tempDir, 'project'); await initKtxProject({ projectDir, projectName: 'warehouse' }); await writeConnections(projectDir, { - warehouse: { driver: 'sqlite', readonly: true }, + warehouse: { driver: 'sqlite' }, }); const cleanup = vi.fn(async () => undefined); const connector: KtxScanConnector = { diff --git a/packages/cli/src/context-build-view.test.ts b/packages/cli/src/context-build-view.test.ts index 03b7b940..5395bd78 100644 --- a/packages/cli/src/context-build-view.test.ts +++ b/packages/cli/src/context-build-view.test.ts @@ -175,6 +175,30 @@ describe('renderContextBuildView', () => { expect(output).toContain('dbt-main'); }); + it('supports text ingest labels while preserving the shared compact progress view', () => { + const state = initViewState([ + { connectionId: 'text-1', driver: 'text', operation: 'source-ingest', debugCommand: '', steps: ['memory-update'] }, + { connectionId: 'schema.md', driver: 'text', operation: 'source-ingest', debugCommand: '', steps: ['memory-update'] }, + ]); + state.contextSources[0].status = 'running'; + state.contextSources[0].detailLine = 'capturing...'; + + const output = renderContextBuildView(state, { + styled: false, + title: 'Ingesting text memory', + contextGroupLabel: 'Texts', + sourceIngestRunningText: 'capturing...', + completedItemName: { singular: 'text', plural: 'texts' }, + }); + + expect(output).toContain('Ingesting text memory'); + expect(output).toContain('Texts:'); + expect(output).toContain('text-1'); + expect(output).toContain('schema.md'); + expect(output).toContain('capturing...'); + expect(output).not.toContain('Context sources:'); + }); + it('renders header with total elapsed time when set', () => { const state = initViewState([ { connectionId: 'warehouse', driver: 'postgres', operation: 'database-ingest', debugCommand: '', steps: ['database-schema'] }, diff --git a/packages/cli/src/context-build-view.ts b/packages/cli/src/context-build-view.ts index 05f73c40..6df8ad2a 100644 --- a/packages/cli/src/context-build-view.ts +++ b/packages/cli/src/context-build-view.ts @@ -115,6 +115,26 @@ export interface ContextBuildSourceProgressUpdate { summaryText?: string; } +interface CompletedItemName { + singular: string; + plural: string; +} + +interface ContextBuildRenderOptions { + styled?: boolean; + showHint?: boolean; + hintText?: string; + projectDir?: string; + title?: string; + primaryGroupLabel?: string; + contextGroupLabel?: string; + scanRunningText?: string; + sourceIngestRunningText?: string; + completedItemName?: CompletedItemName; + notices?: string[]; + warnings?: string[]; +} + export interface ContextBuildDeps { executeTarget?: typeof executePublicIngestTarget; now?: () => number; @@ -224,7 +244,7 @@ function staleProgressText(target: ContextBuildTargetState, styled: boolean): st return styled ? dim(text) : text; } -function targetDetail(target: ContextBuildTargetState, styled: boolean): string { +function targetDetail(target: ContextBuildTargetState, styled: boolean, options: ContextBuildRenderOptions): string { if (target.status === 'done') { const parts: string[] = []; if (target.summaryText) parts.push(target.summaryText); @@ -239,7 +259,9 @@ function targetDetail(target: ContextBuildTargetState, styled: boolean): string const percent = extractPercent(target.detailLine); const progressText = target.detailLine?.replace(/^\[\d+%\]\s*/, '') ?? - (target.target.operation === 'database-ingest' ? 'reading schema' : 'ingesting...'); + (target.target.operation === 'database-ingest' + ? (options.scanRunningText ?? 'reading schema') + : (options.sourceIngestRunningText ?? 'ingesting...')); const elapsed = target.elapsedMs > 0 ? `(${formatDuration(target.elapsedMs)})` : null; const parts: string[] = []; if (percent !== null) { @@ -308,7 +330,13 @@ function columnWidth(state: ContextBuildViewState): number { return Math.max(12, ...all.map((t) => t.target.connectionId.length)) + 2; } -function renderTargetRows(target: ContextBuildTargetState, frame: number, styled: boolean, width: number): string[] { +function renderTargetRows( + target: ContextBuildTargetState, + frame: number, + styled: boolean, + width: number, + options: ContextBuildRenderOptions, +): string[] { const icon = statusIcon(target.status, frame, styled); const name = target.target.connectionId.padEnd(width); const anyPhaseStarted = target.phases.some((p) => p.status !== 'queued'); @@ -317,7 +345,7 @@ function renderTargetRows(target: ContextBuildTargetState, frame: number, styled const headerLine = ` ${icon} ${name} ${headerDetail}`.trimEnd(); return [headerLine, ...target.phases.map((phase) => renderPhaseRow(phase, frame, styled))]; } - return [` ${icon} ${name} ${targetDetail(target, styled)}`]; + return [` ${icon} ${name} ${targetDetail(target, styled, options)}`]; } function renderTargetGroup( @@ -326,9 +354,10 @@ function renderTargetGroup( frame: number, styled: boolean, width: number, + options: ContextBuildRenderOptions, ): string[] { if (targets.length === 0) return []; - return ['', ` ${label}:`, ...targets.flatMap((t) => renderTargetRows(t, frame, styled, width))]; + return ['', ` ${label}:`, ...targets.flatMap((t) => renderTargetRows(t, frame, styled, width, options))]; } function renderMessageGroup(label: string, messages: string[], styled: boolean): string[] { @@ -360,14 +389,7 @@ function retryCommand(input: { export function renderContextBuildView( state: ContextBuildViewState, - options: { - styled?: boolean; - showHint?: boolean; - hintText?: string; - projectDir?: string; - notices?: string[]; - warnings?: string[]; - } = {}, + options: ContextBuildRenderOptions = {}, ): string { const styled = options.styled ?? true; const width = columnWidth(state); @@ -377,7 +399,7 @@ export function renderContextBuildView( const hasActive = allTargets.some((t) => t.status === 'running' || t.status === 'queued'); const allDone = totalCount > 0 && !hasActive; - const headerParts = ['Building KTX context']; + const headerParts = [options.title ?? 'Building KTX context']; if (totalCount > 0) { const progressParts: string[] = [`${doneCount}/${totalCount}`]; if (state.totalElapsedMs > 0) progressParts.push(formatDuration(state.totalElapsedMs)); @@ -393,15 +415,16 @@ export function renderContextBuildView( header, separator, ...(options.projectDir ? [` Project: ${options.projectDir}`] : []), - ...renderTargetGroup('Databases', state.primarySources, state.frame, styled, width), - ...renderTargetGroup('Context sources', state.contextSources, state.frame, styled, width), + ...renderTargetGroup(options.primaryGroupLabel ?? 'Databases', state.primarySources, state.frame, styled, width, options), + ...renderTargetGroup(options.contextGroupLabel ?? 'Context sources', state.contextSources, state.frame, styled, width, options), ...renderMessageGroup('Notices', options.notices ?? [], styled), ...renderMessageGroup('Warnings', options.warnings ?? [], styled), '', ]; if (allDone && state.totalElapsedMs > 0) { - const sourcesLabel = totalCount === 1 ? '1 source' : `${totalCount} sources`; + const itemName = options.completedItemName ?? { singular: 'source', plural: 'sources' }; + const sourcesLabel = totalCount === 1 ? `1 ${itemName.singular}` : `${totalCount} ${itemName.plural}`; const summary = ` Done in ${formatDuration(state.totalElapsedMs)} · ${sourcesLabel} processed`; lines.push(styled ? green(summary) : summary); lines.push(''); diff --git a/packages/cli/src/demo-assets.ts b/packages/cli/src/demo-assets.ts index 1e972ef7..aae9f1a2 100644 --- a/packages/cli/src/demo-assets.ts +++ b/packages/cli/src/demo-assets.ts @@ -57,7 +57,6 @@ function demoConfig(databasePath: string): string { ` ${DEMO_CONNECTION_ID}:`, ' driver: sqlite', ` path: ${JSON.stringify(databasePath)}`, - ' readonly: true', 'storage:', ' state: sqlite', ' search: sqlite-fts5', diff --git a/packages/cli/src/doctor.test.ts b/packages/cli/src/doctor.test.ts index 4373337f..3cbc5bd9 100644 --- a/packages/cli/src/doctor.test.ts +++ b/packages/cli/src/doctor.test.ts @@ -275,7 +275,6 @@ describe('runKtxDoctor', () => { ' warehouse:', ' driver: postgres', ' url: env:WAREHOUSE_DATABASE_URL', - ' readonly: true', ' context:', ' queryHistory:', ' enabled: true', diff --git a/packages/cli/src/historic-sql-doctor.test.ts b/packages/cli/src/historic-sql-doctor.test.ts index fae089e7..d0771348 100644 --- a/packages/cli/src/historic-sql-doctor.test.ts +++ b/packages/cli/src/historic-sql-doctor.test.ts @@ -25,7 +25,7 @@ describe('runPostgresHistoricSqlDoctorChecks', () => { it('passes when no Postgres query-history connections are enabled', async () => { const checks = await runPostgresHistoricSqlDoctorChecks( projectWithConnections({ - warehouse: { driver: 'sqlite', path: './warehouse.db', readonly: true }, + warehouse: { driver: 'sqlite', path: './warehouse.db' }, }), { postgresHistoricSqlProbe: vi.fn(), @@ -53,7 +53,6 @@ describe('runPostgresHistoricSqlDoctorChecks', () => { warehouse: { driver: 'postgres', url: 'env:WAREHOUSE_DATABASE_URL', - readonly: true, context: { queryHistory: { enabled: true } }, }, }), @@ -66,7 +65,6 @@ describe('runPostgresHistoricSqlDoctorChecks', () => { connection: { driver: 'postgres', url: 'env:WAREHOUSE_DATABASE_URL', - readonly: true, context: { queryHistory: { enabled: true } }, }, env: process.env, @@ -87,7 +85,6 @@ describe('runPostgresHistoricSqlDoctorChecks', () => { warehouse: { driver: 'postgres', url: 'env:WAREHOUSE_DATABASE_URL', - readonly: true, context: { queryHistory: { enabled: true } }, }, }), @@ -119,7 +116,6 @@ describe('runPostgresHistoricSqlDoctorChecks', () => { warehouse: { driver: 'postgres', url: 'env:WAREHOUSE_DATABASE_URL', - readonly: true, context: { queryHistory: { enabled: true } }, }, }), @@ -182,7 +178,6 @@ describe('runPostgresHistoricSqlDoctorChecks', () => { warehouse: { driver: 'mysql', url: 'env:WAREHOUSE_DATABASE_URL', - readonly: true, context: { queryHistory: { enabled: true } }, }, }), @@ -208,7 +203,6 @@ describe('runPostgresHistoricSqlDoctorChecks', () => { warehouse: { driver: 'postgres', url: 'env:WAREHOUSE_DATABASE_URL', - readonly: true, context: { queryHistory: { enabled: true } }, }, }), diff --git a/packages/cli/src/historic-sql-doctor.ts b/packages/cli/src/historic-sql-doctor.ts index 30b0e548..2e99975f 100644 --- a/packages/cli/src/historic-sql-doctor.ts +++ b/packages/cli/src/historic-sql-doctor.ts @@ -96,8 +96,9 @@ async function defaultPostgresHistoricSqlProbe( const [{ PostgresPgssReader }, { KtxPostgresHistoricSqlQueryClient, isKtxPostgresConnectionConfig }] = await Promise.all([import('@ktx/context/ingest'), import('@ktx/connector-postgres')]); + const inputDriver = input.connection.driver ?? 'unknown'; if (!isKtxPostgresConnectionConfig(input.connection)) { - throw new Error(`Native PostgreSQL connector cannot run driver "${input.connection.driver ?? 'unknown'}"`); + throw new Error(`Native PostgreSQL connector cannot run driver "${inputDriver}"`); } const client = new KtxPostgresHistoricSqlQueryClient({ diff --git a/packages/cli/src/index.test.ts b/packages/cli/src/index.test.ts index 1f7652fb..ff05a0cc 100644 --- a/packages/cli/src/index.test.ts +++ b/packages/cli/src/index.test.ts @@ -323,6 +323,22 @@ describe('runKtxCli', () => { expect(testIo.stderr()).toBe(`Project: ${tempDir}\n`); }); + it('does not print the command-level project directory line for setup', async () => { + const setup = vi.fn(async () => 0); + const testIo = makeIo(); + + await expect(runKtxCli(['--project-dir', tempDir, 'setup', '--no-input'], testIo.io, { setup })).resolves.toBe(0); + + expect(setup).toHaveBeenCalledWith( + expect.objectContaining({ + command: 'run', + projectDir: tempDir, + }), + testIo.io, + ); + expect(testIo.stderr()).toBe(''); + }); + it('skips the project directory line for JSON output mode', async () => { const publicIngest = vi.fn(async () => 0); const jsonIo = makeIo(); @@ -410,35 +426,62 @@ describe('runKtxCli', () => { expect(io.stderr()).toContain('Choose only one runtime install mode: --yes or --no-input'); }); - it('documents setup as a bare command without subcommands', async () => { + it('documents setup with only the common interactive options visible', async () => { const testIo = makeIo(); await expect(runKtxCli(['setup', '--help'], testIo.io)).resolves.toBe(0); - expect(testIo.stdout()).toContain('Usage: ktx setup [options]'); - expect(testIo.stdout()).not.toContain('Commands:'); - expect(testIo.stdout()).not.toContain('setup demo'); - expect(testIo.stdout()).not.toContain('setup context'); - expect(testIo.stdout()).not.toContain('--skip-llm'); - expect(testIo.stdout()).not.toContain('--skip-embeddings'); - expect(testIo.stdout()).not.toContain('--embedding-model'); - expect(testIo.stdout()).not.toContain('--embedding-dimensions'); - expect(testIo.stdout()).not.toContain('--embedding-base-url'); - for (const expected of [ + const stdout = testIo.stdout(); + expect(stdout).toContain('Usage: ktx setup [options]'); + expect(stdout).toContain('--agents'); + expect(stdout).toContain('--target '); + expect(stdout).toContain('--global'); + expect(stdout).toContain('--yes'); + expect(stdout).toContain('--no-input'); + expect(stdout).toContain('Global Options:'); + expect(stdout.match(/--project-dir /g)).toHaveLength(1); + expect(stdout).not.toContain('Commands:'); + expect(stdout).not.toContain('setup demo'); + expect(stdout).not.toContain('setup context'); + + for (const hiddenFlag of [ + '--new', + '--existing', + '--agent-scope', + '--skip-agents', + '--llm-backend', + '--anthropic-api-key-env', + '--vertex-project', + '--embedding-backend', + '--database ', + '--database-connection-id', + '--new-database-connection-id', + '--enable-historic-sql', + '--historic-sql-min-executions', '--enable-query-history', '--disable-query-history', '--query-history-window-days', '--query-history-min-executions', '--query-history-service-account-pattern', '--query-history-redaction-pattern', + '--skip-databases', + '--source ', + '--source-connection-id', + '--metabase-database-id', + '--notion-root-page-id', + '--skip-initial-source-ingest', + '--skip-sources', + '--skip-llm', + '--skip-embeddings', + '--embedding-model', + '--embedding-dimensions', + '--embedding-base-url', ]) { - expect(testIo.stdout()).toContain(expected); + expect(stdout).not.toContain(hiddenFlag); } - expect(testIo.stdout()).toContain('KTX cannot work until a database is added'); - expect(testIo.stdout()).not.toContain('primary ' + 'source'); - expect(testIo.stdout()).not.toContain('primary ' + 'sources'); - expect(testIo.stdout()).not.toContain('--enable-historic-sql'); - expect(testIo.stdout()).not.toContain('--historic-sql-window-days'); + expect(stdout).not.toMatch(/^ --project\s/m); + expect(stdout).not.toContain('primary ' + 'source'); + expect(stdout).not.toContain('primary ' + 'sources'); expect(testIo.stderr()).toBe(''); }); @@ -737,13 +780,30 @@ describe('runKtxCli', () => { expect(setup).not.toHaveBeenCalled(); }); + it('rejects removed setup options', async () => { + const setup = vi.fn(async () => 0); + const cases = [ + ['setup', '--project'], + ['setup', '--agent-scope', 'global'], + ['setup', '--skip-initial-source-ingest'], + ]; + + for (const args of cases) { + const testIo = makeIo(); + await expect(runKtxCli(['--project-dir', tempDir, ...args], testIo.io, { setup })).resolves.toBe(1); + expect(testIo.stderr()).toMatch(/unknown option|error:/i); + } + + expect(setup).not.toHaveBeenCalled(); + }); + it('prints ingest help without invoking ingest execution', async () => { const testIo = makeIo(); const publicIngest = vi.fn(); await expect(runKtxCli(['ingest', '--help'], testIo.io, { publicIngest })).resolves.toBe(0); - expect(testIo.stdout()).toContain('Usage: ktx ingest [options] [connectionId]'); + expect(testIo.stdout()).toContain('Usage: ktx ingest'); expect(testIo.stdout()).toContain('Build or inspect KTX context'); expect(testIo.stdout()).toContain('--all'); expect(testIo.stdout()).toContain('--fast'); @@ -751,14 +811,73 @@ describe('runKtxCli', () => { expect(testIo.stdout()).toContain('--query-history'); expect(testIo.stdout()).toContain('--no-query-history'); expect(testIo.stdout()).toContain('--query-history-window-days '); + expect(testIo.stdout()).toContain('text'); expect(testIo.stdout()).not.toMatch(/^ status\s/m); expect(testIo.stdout()).not.toMatch(/^ replay\s/m); expect(testIo.stdout()).not.toMatch(/^ run\s/m); expect(testIo.stdout()).not.toMatch(/^ watch\s/m); + expect(testIo.stdout()).not.toContain('--manifest'); expect(testIo.stderr()).toBe(''); expect(publicIngest).not.toHaveBeenCalled(); }); + it('routes text memory ingest through Commander without exposing chat ids', async () => { + const textIngest = vi.fn(async () => 0); + const testIo = makeIo(); + + await expect( + runKtxCli( + [ + '--project-dir', + tempDir, + 'ingest', + 'text', + '--text', + 'Revenue means gross receipts.', + '--text', + 'Orders are completed purchases.', + '--connection-id', + 'warehouse', + '--user-id', + 'agent', + '--json', + '--fail-fast', + ], + testIo.io, + { textIngest }, + ), + ).resolves.toBe(0); + + expect(textIngest).toHaveBeenCalledWith( + { + projectDir: tempDir, + texts: ['Revenue means gross receipts.', 'Orders are completed purchases.'], + files: [], + connectionId: 'warehouse', + userId: 'agent', + json: true, + failFast: true, + }, + testIo.io, + ); + expect(testIo.stderr()).toBe(''); + }); + + it('documents text ingest inputs without a manifest option', async () => { + const textIngest = vi.fn(async () => 0); + const testIo = makeIo(); + + await expect(runKtxCli(['ingest', 'text', '--help'], testIo.io, { textIngest })).resolves.toBe(0); + + expect(testIo.stdout()).toContain('Usage: ktx ingest text [options] [files...]'); + expect(testIo.stdout()).toContain('--text '); + expect(testIo.stdout()).toContain('--connection-id '); + expect(testIo.stdout()).toContain('--user-id '); + expect(testIo.stdout()).toContain('--fail-fast'); + expect(testIo.stdout()).not.toContain('--manifest'); + expect(textIngest).not.toHaveBeenCalled(); + }); + it('rejects old adapter-backed ingest flags at the top level and under dev', async () => { const rootRunIo = makeIo(); const devRunIo = makeIo(); @@ -1142,7 +1261,6 @@ describe('runKtxCli', () => { '--agents', '--target', 'codex', - '--project', '--no-input', '--yes', ], diff --git a/packages/cli/src/local-adapters.test.ts b/packages/cli/src/local-adapters.test.ts index d64f9459..12f8d652 100644 --- a/packages/cli/src/local-adapters.test.ts +++ b/packages/cli/src/local-adapters.test.ts @@ -45,7 +45,6 @@ describe('CLI local ingest adapters', () => { ' warehouse:', ' driver: postgres', ' url: env:WAREHOUSE_DATABASE_URL', - ' readonly: true', ' historicSql:', ' enabled: true', ' dialect: postgres', @@ -108,7 +107,6 @@ describe('CLI local ingest adapters', () => { 'connections:', ' bq:', ' driver: bigquery', - ' readonly: true', ' dataset_id: analytics', ' location: us', ' credentials_json: \'{"project_id":"demo-project"}\'', @@ -142,7 +140,6 @@ describe('CLI local ingest adapters', () => { 'connections:', ' sf:', ' driver: snowflake', - ' readonly: true', ' account: acct', ' warehouse: wh', ' database: ANALYTICS', diff --git a/packages/cli/src/local-adapters.ts b/packages/cli/src/local-adapters.ts index 7edbc5fc..daa8f63b 100644 --- a/packages/cli/src/local-adapters.ts +++ b/packages/cli/src/local-adapters.ts @@ -208,10 +208,9 @@ function enabledHistoricSqlDialect(connection: unknown): 'postgres' | 'bigquery' function createEphemeralPostgresHistoricSqlClient(project: KtxLocalProject, connectionId: string) { const connection = project.config.connections[connectionId] as KtxPostgresConnectionConfig | undefined; + const inputDriver = connection?.driver ?? 'unknown'; if (!isKtxPostgresConnectionConfig(connection)) { - throw new Error( - `Query history ingest requires a Postgres connection, got ${String(connection?.driver ?? 'unknown')}`, - ); + throw new Error(`Query history ingest requires a Postgres connection, got ${String(inputDriver)}`); } return { async executeQuery(sql: string, params?: unknown[]) { @@ -230,10 +229,9 @@ function createEphemeralPostgresHistoricSqlClient(project: KtxLocalProject, conn function createEphemeralBigQueryHistoricSqlClient(project: KtxLocalProject, connectionId: string) { const connection = project.config.connections[connectionId] as KtxBigQueryConnectionConfig | undefined; + const inputDriver = connection?.driver ?? 'unknown'; if (!isKtxBigQueryConnectionConfig(connection)) { - throw new Error( - `Query history ingest requires a BigQuery connection, got ${String(connection?.driver ?? 'unknown')}`, - ); + throw new Error(`Query history ingest requires a BigQuery connection, got ${String(inputDriver)}`); } return { async executeQuery(query: string) { @@ -261,10 +259,9 @@ async function createEphemeralSnowflakeHistoricSqlClient( connectorModule: SnowflakeConnectorModule, ) { const connection = project.config.connections[connectionId]; + const inputDriver = connection?.driver ?? 'unknown'; if (!connectorModule.isKtxSnowflakeConnectionConfig(connection)) { - throw new Error( - `Query history ingest requires a Snowflake connection, got ${String(connection?.driver ?? 'unknown')}`, - ); + throw new Error(`Query history ingest requires a Snowflake connection, got ${String(inputDriver)}`); } return { async executeQuery(query: string) { @@ -326,10 +323,9 @@ function historicSqlOptionsForLocalRun(project: KtxLocalProject, options: KtxCli } if (dialect === 'bigquery') { + const inputDriver = connection?.driver ?? 'unknown'; if (!isKtxBigQueryConnectionConfig(connection)) { - throw new Error( - `Query history ingest requires a BigQuery connection, got ${String(connection?.driver ?? 'unknown')}`, - ); + throw new Error(`Query history ingest requires a BigQuery connection, got ${String(inputDriver)}`); } return { ...base, diff --git a/packages/cli/src/local-scan-connectors.test.ts b/packages/cli/src/local-scan-connectors.test.ts index 087e978d..e8a5c1e9 100644 --- a/packages/cli/src/local-scan-connectors.test.ts +++ b/packages/cli/src/local-scan-connectors.test.ts @@ -49,7 +49,6 @@ describe('createKtxCliScanConnector', () => { ' warehouse:', ' driver: sqlite', ' path: warehouse.db', - ' readonly: true', '', ].join('\n'), 'utf-8', @@ -72,7 +71,6 @@ describe('createKtxCliScanConnector', () => { ' warehouse:', ' driver: bigquery', ' dataset_id: analytics', - ' readonly: true', ' max_bytes_billed: "987654321"', '', ].join('\n'), @@ -123,7 +121,6 @@ describe('createKtxCliScanConnector', () => { ' warehouse:', ' type: postgres', ' url: postgresql://example/db', - ' readonly: true', '', ].join('\n'), 'utf-8', diff --git a/packages/cli/src/notion-page-picker-tree.test.ts b/packages/cli/src/notion-page-picker-tree.test.ts index 94b46b57..58e8c7ca 100644 --- a/packages/cli/src/notion-page-picker-tree.test.ts +++ b/packages/cli/src/notion-page-picker-tree.test.ts @@ -11,7 +11,6 @@ import { selectAllVisible, selectNone, toggleChecked, - TRANSIENT_HINT_DURATION_MS, visibleNodeIds, type NotionPickerPageInput, } from './notion-page-picker-tree.js'; @@ -223,22 +222,24 @@ describe('bulk actions and reducer effects', () => { }); }); - it('blocks empty saves, updates search state, and quits without saving', () => { + it('prompts skip-empty confirmation on empty save, updates search state, and quits without saving', () => { const state = buildInitialState({ tree: buildPickerTree(pages()), existingRootPageIds: [], currentCrawlMode: 'selected_roots', }); - const blockedSave = reducer(state, 'save-request', 9000); - expect(blockedSave).toEqual({ - next: { - ...state, - transientHint: { - text: 'Select at least one page or press q to quit', - expiresAt: 9000 + TRANSIENT_HINT_DURATION_MS, - }, - }, + const emptySave = reducer(state, 'save-request'); + expect(emptySave).toEqual({ + next: { ...state, pendingConfirm: 'skip-empty' }, + effect: null, + }); + expect(reducer(emptySave.next, 'save-confirm')).toEqual({ + next: { ...state, pendingConfirm: null }, + effect: 'quit-without-save', + }); + expect(reducer(emptySave.next, 'save-cancel')).toEqual({ + next: { ...state, pendingConfirm: null }, effect: null, }); expect( @@ -262,7 +263,7 @@ describe('bulk actions and reducer effects', () => { const withHint = { ...state, transientHint: { - text: 'Select at least one page or press q to quit', + text: 'Select at least one page or press esc to cancel', expiresAt: 11500, }, }; diff --git a/packages/cli/src/notion-page-picker-tree.ts b/packages/cli/src/notion-page-picker-tree.ts index 379ac938..738ab723 100644 --- a/packages/cli/src/notion-page-picker-tree.ts +++ b/packages/cli/src/notion-page-picker-tree.ts @@ -22,7 +22,7 @@ export interface PickerState { checked: Set; cursorId: string; search: { editing: boolean; query: string }; - pendingConfirm: 'mode-switch' | null; + pendingConfirm: 'mode-switch' | 'skip-empty' | null; preLoadWarnings: string[]; transientHint: { text: string; expiresAt: number } | null; currentCrawlMode: 'all_accessible' | 'selected_roots'; @@ -61,7 +61,7 @@ interface MutableNode { childIds: string[]; } -export const TRANSIENT_HINT_DURATION_MS = 2500; +const TRANSIENT_HINT_DURATION_MS = 2500; const collator = new Intl.Collator('en', { sensitivity: 'base', numeric: true }); @@ -444,7 +444,8 @@ export function buildInitialState(args: { export function reducer(state: PickerState, cmd: PickerCommand, now = Date.now()): { next: PickerState; effect: PickerEffect } { if (state.pendingConfirm) { if (cmd === 'save-confirm') { - return { next: cloneState(state, { pendingConfirm: null }), effect: 'save' }; + const effect: PickerEffect = state.pendingConfirm === 'skip-empty' ? 'quit-without-save' : 'save'; + return { next: cloneState(state, { pendingConfirm: null }), effect }; } if (cmd === 'save-cancel') { return { next: cloneState(state, { pendingConfirm: null }), effect: null }; @@ -498,19 +499,13 @@ export function reducer(state: PickerState, cmd: PickerCommand, now = Date.now() }; case 'save-request': if (state.checked.size === 0) { - return { - next: cloneState(state, { - transientHint: transientHint('Select at least one page or press q to quit', now), - }), - effect: null, - }; + return { next: cloneState(state, { pendingConfirm: 'skip-empty' }), effect: null }; } if (state.currentCrawlMode === 'all_accessible') { return { next: cloneState(state, { pendingConfirm: 'mode-switch' }), effect: null }; } return { next: state, effect: 'save' }; case 'save-confirm': - return { next: state, effect: 'save' }; case 'save-cancel': return { next: state, effect: null }; case 'quit': diff --git a/packages/cli/src/notion-page-picker-tui.test.tsx b/packages/cli/src/notion-page-picker-tui.test.tsx index 2d4dffc3..16ad93db 100644 --- a/packages/cli/src/notion-page-picker-tui.test.tsx +++ b/packages/cli/src/notion-page-picker-tui.test.tsx @@ -1,7 +1,7 @@ /* @jsxImportSource react */ import { render as renderInkTest } from 'ink-testing-library'; -import { act, type ReactNode } from 'react'; -import { afterEach, describe, expect, it, vi } from 'vitest'; +import { type ReactNode } from 'react'; +import { describe, expect, it, vi } from 'vitest'; import { buildInitialState, buildPickerTree, type NotionPickerPageInput } from './notion-page-picker-tree.js'; import { NotionPickerApp, @@ -70,13 +70,9 @@ function fakeInkInstance(): NotionPickerInkInstance { } function normalizeFrameWrap(frame: string | undefined): string { - return frame?.replace(/\n/g, ' ') ?? ''; + return frame?.replace(/\n/g, ' ').replace(/│ /g, '').replace(/ +/g, ' ') ?? ''; } -afterEach(() => { - vi.useRealTimers(); -}); - describe('notionPickerCommandForInkInput', () => { it('maps browse, search, and confirm input to reducer commands', () => { expect(notionPickerCommandForInkInput('', { downArrow: true }, state().search, null)).toBe('cursor-down'); @@ -87,9 +83,11 @@ describe('notionPickerCommandForInkInput', () => { expect(notionPickerCommandForInkInput('/', {}, state().search, null)).toBe('search-start'); expect(notionPickerCommandForInkInput('a', {}, state().search, null)).toBe('select-all-visible'); expect(notionPickerCommandForInkInput('n', {}, state().search, null)).toBe('select-none'); - expect(notionPickerCommandForInkInput('s', {}, state().search, null)).toBe('save-request'); - expect(notionPickerCommandForInkInput('q', {}, state().search, null)).toBe('quit'); + expect(notionPickerCommandForInkInput('', { return: true }, state().search, null)).toBe('save-request'); + expect(notionPickerCommandForInkInput('', { escape: true }, state().search, null)).toBe('quit'); expect(notionPickerCommandForInkInput('c', { ctrl: true }, state().search, null)).toBe('quit'); + expect(notionPickerCommandForInkInput('s', {}, state().search, null)).toBeNull(); + expect(notionPickerCommandForInkInput('q', {}, state().search, null)).toBeNull(); expect(notionPickerCommandForInkInput('x', {}, { editing: true, query: '' }, null)).toEqual({ type: 'search-input', @@ -145,13 +143,16 @@ describe('NotionPickerApp', () => { ); const frame = lastFrame() ?? ''; - expect(frame).toContain('Notion pages visible to integration "Design Workspace"'); + expect(frame).toContain('Select Notion pages to ingest'); + expect(frame).toContain('Workspace: Design Workspace'); expect(frame).toContain('5000-page cap reached - some pages not shown'); expect(frame).toContain('1 stored root_page_ids no longer visible - they will be removed if you save'); - expect(frame).toContain('▸ [ ] Engineering Docs ▸ (1)'); - expect(frame).toContain(' [ ] Marketing'); + expect(frame).toContain('◻ Engineering Docs ▸ (1)'); + expect(frame).toContain('◻ Marketing'); expect(frame).not.toContain('Search ready: -'); - expect(frame).toContain('space toggle · enter expand · / search · a all · n none · s save & exit · q quit'); + expect(normalizeFrameWrap(frame)).toContain( + 'Right Arrow to expand, Up/Down to move, Space to select or unselect, Slash to filter, Enter to confirm, Escape to go back, or Ctrl+C to exit.', + ); }); it('renders partial discovery warnings without stale-root save suffix', () => { @@ -199,8 +200,8 @@ describe('NotionPickerApp', () => { ); const frame = lastFrame() ?? ''; - expect(frame).toContain('▸ [×] Engineering Docs ▾'); - expect(frame).toContain(' [~] Architecture'); + expect(frame).toContain('◼ Engineering Docs ▾'); + expect(frame).toContain(' ◼ Architecture'); }); it('supports keyboard selection, all_accessible confirmation, and save callback', async () => { @@ -220,12 +221,12 @@ describe('NotionPickerApp', () => { stdin.write(' '); await waitForInkInput(); - expect(lastFrame()).toContain('[×] Engineering Docs'); + expect(lastFrame()).toContain('◼ Engineering Docs'); - stdin.write('s'); + stdin.write('\r'); await waitForInkInput(); expect(normalizeFrameWrap(lastFrame())).toContain( - 'Save will switch crawl_mode all_accessible -> selected_roots and limit ingest to 1 selected page. [y] confirm [esc] back', + 'Switch crawl_mode from all_accessible to selected_roots? Will limit ingest to 1 selected page. Press Enter to confirm or Escape to go back.', ); stdin.write('y'); @@ -233,8 +234,7 @@ describe('NotionPickerApp', () => { expect(onExit).toHaveBeenCalledWith({ kind: 'save', rootPageIds: [IDS.engineering] }); }); - it('removes transient hints after their expiry time', async () => { - vi.useFakeTimers(); + it('prompts skip-empty confirmation on empty submit and dismisses on cancel', async () => { const onExit = vi.fn(); const { stdin, lastFrame } = renderInkTest( { />, ); - await act(async () => { - stdin.write('s'); - await vi.advanceTimersByTimeAsync(10); - }); - expect(lastFrame()).toContain('Select at least one page or press q to quit'); - - await act(async () => { - await vi.advanceTimersByTimeAsync(2500); - }); - expect(lastFrame()).not.toContain('Select at least one page or press q to quit'); + stdin.write('\r'); + await waitForInkInput(); + expect(normalizeFrameWrap(lastFrame())).toContain( + 'Nothing selected. Skip this step? Press Enter to skip or Escape to go back.', + ); expect(onExit).not.toHaveBeenCalled(); + + stdin.write('n'); + await waitForInkInput(); + expect(lastFrame()).not.toContain('Nothing selected. Skip this step?'); + expect(onExit).not.toHaveBeenCalled(); + + stdin.write('\r'); + await waitForInkInput(); + expect(lastFrame()).toContain('Nothing selected. Skip this step?'); + + stdin.write('\r'); + await waitForInkInput(); + expect(onExit).toHaveBeenCalledWith({ kind: 'quit' }); }); it('renders row-window overflow indicators when the visible list is clipped', async () => { @@ -312,7 +320,7 @@ describe('NotionPickerApp', () => { />, ); - stdin.write('q'); + stdin.write('\u0003'); await waitForInkInput(); expect(onExit).toHaveBeenCalledWith({ kind: 'quit' }); }); diff --git a/packages/cli/src/notion-page-picker-tui.tsx b/packages/cli/src/notion-page-picker-tui.tsx index 30af7522..d627d200 100644 --- a/packages/cli/src/notion-page-picker-tui.tsx +++ b/packages/cli/src/notion-page-picker-tui.tsx @@ -16,6 +16,7 @@ const COLOR_THEME = { text: 'white', muted: 'gray', active: 'cyan', + selected: 'green', warning: 'yellow', } as const; @@ -23,6 +24,7 @@ const NO_COLOR_THEME = { text: 'white', muted: 'white', active: 'white', + selected: 'white', warning: 'white', } as const; @@ -158,13 +160,12 @@ export function notionPickerCommandForInkInput( if (key.downArrow) return 'cursor-down'; if (key.leftArrow) return 'cursor-left'; if (key.rightArrow) return 'cursor-right'; - if (key.return) return 'expand'; + if (key.return) return 'save-request'; if (input === ' ') return 'toggle-check'; if (input === '/') return 'search-start'; if (input === 'a') return 'select-all-visible'; if (input === 'n') return 'select-none'; - if (input === 's') return 'save-request'; - if (input === 'q' || key.escape) return 'quit'; + if (key.escape) return 'quit'; return null; } @@ -174,18 +175,27 @@ function PickerRow(props: { state: PickerState; nodeId: string; width: number; t const focused = props.state.cursorId === node.id; const locked = isAncestorChecked(node.id, props.state.checked, props.state.byId); const checked = props.state.checked.has(node.id); - const glyph = locked ? '[~]' : checked ? '[×]' : '[ ]'; - const children = + const isSelected = checked || locked; + const glyph = isSelected ? '◼' : '◻'; + const glyphColor = locked ? props.theme.muted : checked ? props.theme.selected : props.theme.muted; + const childAffordance = node.childIds.length > 0 ? (props.state.expanded.has(node.id) ? ' ▾' : ` ▸ (${node.childIds.length})`) : ''; - const prefix = `${focused ? '▸' : ' '} ${glyph} ${' '.repeat(node.depth * 2)}`; - const color = focused ? props.theme.active : locked || node.archived ? props.theme.muted : props.theme.text; - const title = truncateText(`${node.title}${children}`, Math.max(10, props.width - prefix.length)); + const indent = ' '.repeat(node.depth * 2); + const titleColor = focused ? props.theme.text : props.theme.muted; const inverse = rowMatchesSearch(props.state, node.id); + const prefixWidth = indent.length + 2; + const title = truncateText(`${node.title}${childAffordance}`, Math.max(10, props.width - prefixWidth)); return ( - - {prefix} - {title} + + + {indent} + {glyph} + + + {' '} + {title} + ); } @@ -198,7 +208,7 @@ export function NotionPickerApp(props: NotionPickerAppProps): ReactNode { const visibleIds = visibleNodeIds(state); const selectedIndex = Math.max(0, visibleIds.indexOf(state.cursorId)); const reservedRows = state.pendingConfirm === 'mode-switch' ? 9 : 8; - const visibleRows = Math.max(5, Math.min(20, (props.terminalRows ?? 24) - reservedRows)); + const visibleRows = Math.max(5, Math.min(12, (props.terminalRows ?? 24) - reservedRows)); const rows = windowItems(visibleIds, selectedIndex, visibleRows); const hiddenAbove = rows.offset; const hiddenBelow = Math.max(0, visibleIds.length - rows.offset - rows.items.length); @@ -254,34 +264,60 @@ export function NotionPickerApp(props: NotionPickerAppProps): ReactNode { return ( - Notion pages visible to integration "{props.workspaceLabel}" - {props.cappedAtCount ? {props.cappedAtCount}-page cap reached - some pages not shown : null} - {state.preLoadWarnings.map((warning) => ( - - {staleWarningText(warning)} - - ))} - {showSearch ? ( + + ◆ + Select Notion pages to ingest + + - / {state.search.query} - {state.search.editing ? '█' : ''} ({searchMatchCount} matches) + Right Arrow to expand, Up/Down to move, Space to select or unselect, Slash to filter, Enter to confirm, Escape + to go back, or Ctrl+C to exit. - ) : null} - + + Workspace: {props.workspaceLabel} + {props.cappedAtCount ? ( + {props.cappedAtCount}-page cap reached - some pages not shown + ) : null} + {state.preLoadWarnings.map((warning) => ( + + {staleWarningText(warning)} + + ))} + {showSearch ? ( + + / + + {state.search.query} + {state.search.editing ? '█' : ''} + + ({searchMatchCount} matches) + + ) : null} {hiddenAbove > 0 ? ↑ {hiddenAbove} more : null} {rows.items.map((nodeId) => ( ))} {hiddenBelow > 0 ? ↓ {hiddenBelow} more : null} + {state.pendingConfirm === 'mode-switch' ? ( + + Switch crawl_mode from all_accessible to selected_roots? Will limit ingest to{' '} + {selectedPageCountText(selectedCount)}. Press Enter to confirm or Escape to go back. + + ) : null} + {state.pendingConfirm === 'skip-empty' ? ( + Nothing selected. Skip this step? Press Enter to skip or Escape to go back. + ) : null} + {state.transientHint ? {state.transientHint.text} : null} - {state.pendingConfirm === 'mode-switch' ? ( - - Save will switch crawl_mode all_accessible -> selected_roots and limit ingest to{' '} - {selectedPageCountText(selectedCount)}. [y] confirm [esc] back - - ) : null} - {state.transientHint ? {state.transientHint.text} : null} - space toggle · enter expand · / search · a all · n none · s save & exit · q quit + └ ); } @@ -323,7 +359,7 @@ export async function renderNotionPickerTui( exitOnCtrlC: false, patchConsole: false, maxFps: 30, - alternateScreen: true, + alternateScreen: false, }, ); await instance.waitUntilExit(); diff --git a/packages/cli/src/project-dir.test.ts b/packages/cli/src/project-dir.test.ts index 9a6d0c5a..7d25e56d 100644 --- a/packages/cli/src/project-dir.test.ts +++ b/packages/cli/src/project-dir.test.ts @@ -58,7 +58,7 @@ describe('project directory defaults', () => { argv: ['setup', '--no-input'], spy: setup, expected: { command: 'run', projectDir: '/tmp/ktx-env-project' }, - expectedStderr: 'Project: /tmp/ktx-env-project\n', + expectedStderr: '', }, { argv: ['ingest', 'warehouse', '--no-input'], diff --git a/packages/cli/src/scan.test.ts b/packages/cli/src/scan.test.ts index 28c60ea0..c4cbaf70 100644 --- a/packages/cli/src/scan.test.ts +++ b/packages/cli/src/scan.test.ts @@ -861,7 +861,6 @@ describe('runKtxScan', () => { ' warehouse:', ' driver: mysql', ' url: env:MYSQL_URL', - ' readonly: true', '', ].join('\n'), 'utf-8', @@ -910,7 +909,6 @@ describe('runKtxScan', () => { ' warehouse:', ' driver: sqlite', ' path: warehouse.db', - ' readonly: true', '', ].join('\n'), 'utf-8', @@ -968,7 +966,6 @@ describe('runKtxScan', () => { ' database: analytics', ' username: reader', ' password: env:POSTGRES_PASSWORD', - ' readonly: true', '', ].join('\n'), 'utf-8', @@ -1035,7 +1032,6 @@ describe('runKtxScan', () => { ' database: analytics', ' username: reader', ' password: env:CLICKHOUSE_PASSWORD', - ' readonly: true', '', ].join('\n'), 'utf-8', @@ -1087,7 +1083,6 @@ describe('runKtxScan', () => { ' database: analytics', ' username: reader', ' schema: dbo', - ' readonly: true', '', ].join('\n'), 'utf-8', @@ -1153,7 +1148,6 @@ describe('runKtxScan', () => { ' dataset_id: analytics', ' credentials_json: env:BIGQUERY_CREDENTIALS_JSON', ' location: US', - ' readonly: true', '', ].join('\n'), 'utf-8', @@ -1222,7 +1216,6 @@ describe('runKtxScan', () => { ' database: ANALYTICS', ' schema_name: PUBLIC', ' username: reader', - ' readonly: true', '', ].join('\n'), 'utf-8', diff --git a/packages/cli/src/setup-agents.ts b/packages/cli/src/setup-agents.ts index 9505307d..7a18a969 100644 --- a/packages/cli/src/setup-agents.ts +++ b/packages/cli/src/setup-agents.ts @@ -82,7 +82,7 @@ export function plannedKtxAgentFiles(input: { { kind: 'file', path: join(codexHome, 'instructions/ktx.md'), role: 'rule' as const }, ]; } - throw new Error(`Global ${input.target} installation is not supported; use --project.`); + throw new Error(`Global ${input.target} installation is not supported; omit --global.`); } const root = resolve(input.projectDir); diff --git a/packages/cli/src/setup-databases.test.ts b/packages/cli/src/setup-databases.test.ts index a70876fb..9d388c8f 100644 --- a/packages/cli/src/setup-databases.test.ts +++ b/packages/cli/src/setup-databases.test.ts @@ -226,7 +226,6 @@ describe('setup databases step', () => { ' warehouse:', ' driver: postgres', ' url: env:DATABASE_URL', - ' readonly: true', '', ].join('\n'), 'utf-8', @@ -249,8 +248,9 @@ describe('setup databases step', () => { expect(prompts.select).toHaveBeenCalledWith({ message: 'Configure PostgreSQL', options: [ - { value: 'existing:warehouse', label: 'Use existing PostgreSQL connection: warehouse' }, - { value: 'new', label: 'Add new PostgreSQL connection' }, + { value: 'existing:warehouse', label: 'Keep existing PostgreSQL connection: warehouse' }, + { value: 'edit:warehouse', label: 'Edit PostgreSQL connection: warehouse' }, + { value: 'new', label: 'Add another PostgreSQL connection' }, { value: 'back', label: 'Back' }, ], }); @@ -290,7 +290,6 @@ describe('setup databases step', () => { expect(config.connections['postgres-warehouse']).toEqual({ driver: 'postgres', url: 'env:DATABASE_URL', - readonly: true, context: { depth: 'fast' }, }); }); @@ -560,7 +559,6 @@ describe('setup databases step', () => { ' warehouse:', ' driver: postgres', ' url: env:DATABASE_URL', - ' readonly: true', 'setup:', ' database_connection_ids:', ' - warehouse', @@ -591,6 +589,7 @@ describe('setup databases step', () => { message: 'Databases already configured: warehouse\nWhat would you like to do?', options: [ { value: 'continue', label: 'Continue to context sources' }, + { value: 'edit', label: 'Edit an existing database' }, { value: 'add', label: 'Add another database' }, ], }); @@ -607,7 +606,6 @@ describe('setup databases step', () => { ' warehouse:', ' driver: postgres', ' url: env:DATABASE_URL', - ' readonly: true', 'setup:', ' database_connection_ids:', ' - warehouse', @@ -642,10 +640,15 @@ describe('setup databases step', () => { connectionIds: ['warehouse', 'mysql-warehouse'], }); expect(prompts.multiselect).toHaveBeenCalledTimes(1); + expect(prompts.multiselect).toHaveBeenCalledWith(expect.objectContaining({ + initialValues: ['postgres'], + required: true, + })); expect(prompts.select).toHaveBeenCalledWith({ message: 'Databases already configured: warehouse\nWhat would you like to do?', options: [ { value: 'continue', label: 'Continue to context sources' }, + { value: 'edit', label: 'Edit an existing database' }, { value: 'add', label: 'Add another database' }, ], }); @@ -683,10 +686,15 @@ describe('setup databases step', () => { connectionIds: ['postgres-warehouse', 'mysql-warehouse'], }); expect(prompts.multiselect).toHaveBeenCalledTimes(2); + expect(prompts.multiselect).toHaveBeenNthCalledWith(2, expect.objectContaining({ + initialValues: ['postgres'], + required: true, + })); expect(prompts.select).toHaveBeenCalledWith({ message: 'Databases already configured: postgres-warehouse\nWhat would you like to do?', options: [ { value: 'continue', label: 'Continue to context sources' }, + { value: 'edit', label: 'Edit an existing database' }, { value: 'add', label: 'Add another database' }, ], }); @@ -722,11 +730,16 @@ describe('setup databases step', () => { connectionIds: ['postgres-warehouse'], }); expect(prompts.multiselect).toHaveBeenCalledTimes(2); + expect(prompts.multiselect).toHaveBeenNthCalledWith(2, expect.objectContaining({ + initialValues: ['postgres'], + required: true, + })); expect(io.stdout()).not.toContain('KTX cannot work without at least one database'); expect(prompts.select).toHaveBeenNthCalledWith(3, { message: 'Databases already configured: postgres-warehouse\nWhat would you like to do?', options: [ { value: 'continue', label: 'Continue to context sources' }, + { value: 'edit', label: 'Edit an existing database' }, { value: 'add', label: 'Add another database' }, ], }); @@ -741,7 +754,6 @@ describe('setup databases step', () => { ' warehouse:', ' driver: postgres', ' url: env:DATABASE_URL', - ' readonly: true', 'setup:', ' database_connection_ids:', ' - warehouse', @@ -763,16 +775,394 @@ describe('setup databases step', () => { ); expect(result).toEqual({ status: 'ready', projectDir: tempDir, connectionIds: ['warehouse'] }); + expect(prompts.multiselect).toHaveBeenCalledWith(expect.objectContaining({ + initialValues: ['postgres'], + required: true, + })); expect(io.stdout()).not.toContain('KTX cannot work without at least one database'); expect(prompts.select).toHaveBeenNthCalledWith(2, { message: 'Databases already configured: warehouse\nWhat would you like to do?', options: [ { value: 'continue', label: 'Continue to context sources' }, + { value: 'edit', label: 'Edit an existing database' }, { value: 'add', label: 'Add another database' }, ], }); }); + it('returns from database edit selection back to the configured source menu', async () => { + await writeFile( + join(tempDir, 'ktx.yaml'), + [ + 'project: warehouse', + 'connections:', + ' warehouse:', + ' driver: postgres', + ' url: env:DATABASE_URL', + 'setup:', + ' database_connection_ids:', + ' - warehouse', + '', + ].join('\n'), + 'utf-8', + ); + await writeKtxSetupState(tempDir, { completed_steps: ['databases'] }); + const prompts = makePromptAdapter({ + selectValues: ['edit', 'back', 'continue'], + }); + const testConnection = vi.fn(async () => 0); + const scanConnection = vi.fn(async () => 0); + + const result = await runKtxSetupDatabasesStep( + { projectDir: tempDir, inputMode: 'auto', skipDatabases: false, databaseSchemas: [] }, + makeIo().io, + { prompts, testConnection, scanConnection }, + ); + + expect(result).toEqual({ status: 'ready', projectDir: tempDir, connectionIds: ['warehouse'] }); + expect(prompts.select).toHaveBeenNthCalledWith(2, { + message: 'Database to edit', + options: [ + { value: 'warehouse', label: 'warehouse (PostgreSQL)' }, + { value: 'back', label: 'Back' }, + ], + }); + expect(prompts.select).toHaveBeenNthCalledWith(3, { + message: 'Databases already configured: warehouse\nWhat would you like to do?', + options: [ + { value: 'continue', label: 'Continue to context sources' }, + { value: 'edit', label: 'Edit an existing database' }, + { value: 'add', label: 'Add another database' }, + ], + }); + expect(testConnection).not.toHaveBeenCalled(); + expect(scanConnection).not.toHaveBeenCalled(); + }); + + it('reruns table selection after editing schema scope so stale enabled tables are removed', async () => { + await writeFile( + join(tempDir, 'ktx.yaml'), + [ + 'project: warehouse', + 'connections:', + ' warehouse:', + ' driver: postgres', + ' url: env:DATABASE_URL', + ' schemas:', + ' - public', + ' enabled_tables:', + ' - public.orders', + 'setup:', + ' database_connection_ids:', + ' - warehouse', + '', + ].join('\n'), + 'utf-8', + ); + await writeKtxSetupState(tempDir, { completed_steps: ['databases'] }); + const prompts = makePromptAdapter({ + textValues: ['env:DATABASE_URL'], + multiselectValues: [['analytics']], + }); + let primaryMenuCount = 0; + vi.mocked(prompts.select).mockImplementation(async (options) => { + if (options.message === 'Databases already configured: warehouse\nWhat would you like to do?') { + primaryMenuCount += 1; + return primaryMenuCount === 1 ? 'edit' : 'continue'; + } + if (options.message === 'Database to edit') return 'warehouse'; + if (options.message === 'How do you want to connect to PostgreSQL?') return 'url'; + if (options.message.startsWith('Enable query-history ingest')) return 'no'; + return 'back'; + }); + const testConnection = vi.fn(async () => 0); + const scanConnection = vi.fn(async () => 0); + const listSchemas = vi.fn(async () => ['analytics', 'public']); + const listTables = vi.fn(async () => [{ schema: 'analytics', name: 'customers', kind: 'table' as const }]); + + const result = await runKtxSetupDatabasesStep( + { projectDir: tempDir, inputMode: 'auto', skipDatabases: false, databaseSchemas: [] }, + makeIo().io, + { prompts, testConnection, scanConnection, listSchemas, listTables }, + ); + + expect(result).toEqual({ status: 'ready', projectDir: tempDir, connectionIds: ['warehouse'] }); + expect(prompts.text).toHaveBeenCalledWith({ + message: textInputPrompt('PostgreSQL connection URL'), + placeholder: 'env:DATABASE_URL', + initialValue: 'env:DATABASE_URL', + }); + expect(listTables).toHaveBeenCalledWith(tempDir, 'warehouse'); + expect(testConnection).toHaveBeenCalledWith(tempDir, 'warehouse', expect.anything()); + expect(scanConnection).toHaveBeenCalledWith(tempDir, 'warehouse', expect.anything()); + const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + expect(config.connections.warehouse).toMatchObject({ + schemas: ['analytics'], + enabled_tables: ['analytics.customers'], + }); + }); + + it('preselects existing schema and table choices when editing a database', async () => { + await writeFile( + join(tempDir, 'ktx.yaml'), + [ + 'project: warehouse', + 'connections:', + ' warehouse:', + ' driver: postgres', + ' url: env:DATABASE_URL', + ' schemas:', + ' - public', + ' enabled_tables:', + ' - public.customers', + ' - public.orders', + 'setup:', + ' database_connection_ids:', + ' - warehouse', + '', + ].join('\n'), + 'utf-8', + ); + await writeKtxSetupState(tempDir, { completed_steps: ['databases'] }); + const prompts = makePromptAdapter({ + textValues: ['env:DATABASE_URL'], + multiselectValues: [['public'], ['public.customers', 'public.orders']], + }); + let primaryMenuCount = 0; + vi.mocked(prompts.select).mockImplementation(async (options) => { + if (options.message === 'Databases already configured: warehouse\nWhat would you like to do?') { + primaryMenuCount += 1; + return primaryMenuCount === 1 ? 'edit' : 'continue'; + } + if (options.message === 'Database to edit') return 'warehouse'; + if (options.message === 'How do you want to connect to PostgreSQL?') return 'url'; + if (options.message.startsWith('Enable query-history ingest')) return 'no'; + if (options.message.startsWith('Tables found in selected schemas')) return 'customize'; + return 'back'; + }); + const listSchemas = vi.fn(async () => ['orbit_analytics', 'orbit_raw', 'public']); + const listTables = vi.fn(async () => [ + { schema: 'public', name: 'customers', kind: 'table' as const }, + { schema: 'public', name: 'orders', kind: 'table' as const }, + { schema: 'public', name: 'products', kind: 'table' as const }, + ]); + + const result = await runKtxSetupDatabasesStep( + { projectDir: tempDir, inputMode: 'auto', skipDatabases: false, databaseSchemas: [] }, + makeIo().io, + { + prompts, + testConnection: vi.fn(async () => 0), + scanConnection: vi.fn(async () => 0), + listSchemas, + listTables, + }, + ); + + expect(result).toEqual({ status: 'ready', projectDir: tempDir, connectionIds: ['warehouse'] }); + expect(prompts.multiselect).toHaveBeenNthCalledWith(1, { + message: expect.stringContaining('PostgreSQL schemas to include'), + options: [ + { value: 'orbit_analytics', label: 'orbit_analytics' }, + { value: 'orbit_raw', label: 'orbit_raw' }, + { value: 'public', label: 'public' }, + ], + initialValues: ['public'], + required: true, + }); + expect(prompts.multiselect).toHaveBeenNthCalledWith(2, { + message: expect.stringContaining('Tables to enable for warehouse'), + options: [ + { value: 'public.customers', label: 'public.customers' }, + { value: 'public.orders', label: 'public.orders' }, + { value: 'public.products', label: 'public.products' }, + ], + initialValues: ['public.customers', 'public.orders'], + required: true, + }); + const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + expect(config.connections.warehouse).toMatchObject({ + schemas: ['public'], + enabled_tables: ['public.customers', 'public.orders'], + }); + }); + + it('returns to the configured primary menu when backing out of schema review during edit', async () => { + await writeFile( + join(tempDir, 'ktx.yaml'), + [ + 'project: warehouse', + 'connections:', + ' warehouse:', + ' driver: postgres', + ' url: env:DATABASE_URL', + ' schemas:', + ' - public', + ' enabled_tables:', + ' - public.orders', + 'setup:', + ' database_connection_ids:', + ' - warehouse', + '', + ].join('\n'), + 'utf-8', + ); + await writeKtxSetupState(tempDir, { completed_steps: ['databases'] }); + const prompts = makePromptAdapter({ + textValues: ['env:DATABASE_URL'], + multiselectValues: [['back']], + }); + let primaryMenuCount = 0; + vi.mocked(prompts.select).mockImplementation(async (options) => { + if (options.message === 'Databases already configured: warehouse\nWhat would you like to do?') { + primaryMenuCount += 1; + return primaryMenuCount === 1 ? 'edit' : 'continue'; + } + if (options.message === 'Database to edit') return 'warehouse'; + if (options.message === 'How do you want to connect to PostgreSQL?') return 'url'; + if (options.message.startsWith('Enable query-history ingest')) return 'no'; + return 'back'; + }); + const testConnection = vi.fn(async () => 0); + const scanConnection = vi.fn(async () => 0); + const listSchemas = vi.fn(async () => ['analytics', 'public']); + const listTables = vi.fn(async () => [{ schema: 'analytics', name: 'customers', kind: 'table' as const }]); + + const result = await runKtxSetupDatabasesStep( + { projectDir: tempDir, inputMode: 'auto', skipDatabases: false, databaseSchemas: [] }, + makeIo().io, + { prompts, testConnection, scanConnection, listSchemas, listTables }, + ); + + expect(result).toEqual({ status: 'ready', projectDir: tempDir, connectionIds: ['warehouse'] }); + expect(primaryMenuCount).toBe(2); + expect(testConnection).toHaveBeenCalledWith(tempDir, 'warehouse', expect.anything()); + expect(scanConnection).not.toHaveBeenCalled(); + expect(listTables).not.toHaveBeenCalled(); + const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + expect(config.connections.warehouse).toMatchObject({ + url: 'env:DATABASE_URL', + schemas: ['public'], + enabled_tables: ['public.orders'], + }); + }); + + it('returns to the configured primary menu when backing out of table review during edit', async () => { + await writeFile( + join(tempDir, 'ktx.yaml'), + [ + 'project: warehouse', + 'connections:', + ' warehouse:', + ' driver: postgres', + ' url: env:DATABASE_URL', + ' schemas:', + ' - public', + ' enabled_tables:', + ' - public.orders', + 'setup:', + ' database_connection_ids:', + ' - warehouse', + '', + ].join('\n'), + 'utf-8', + ); + await writeKtxSetupState(tempDir, { completed_steps: ['databases'] }); + const prompts = makePromptAdapter({ textValues: ['env:DATABASE_URL'] }); + let primaryMenuCount = 0; + vi.mocked(prompts.select).mockImplementation(async (options) => { + if (options.message === 'Databases already configured: warehouse\nWhat would you like to do?') { + primaryMenuCount += 1; + return primaryMenuCount === 1 ? 'edit' : 'continue'; + } + if (options.message === 'Database to edit') return 'warehouse'; + if (options.message === 'How do you want to connect to PostgreSQL?') return 'url'; + if (options.message.startsWith('Enable query-history ingest')) return 'no'; + if (options.message.startsWith('Tables found in selected schemas')) return 'back'; + return 'back'; + }); + const testConnection = vi.fn(async () => 0); + const scanConnection = vi.fn(async () => 0); + const listSchemas = vi.fn(async () => ['public']); + const listTables = vi.fn(async () => [ + { schema: 'public', name: 'customers', kind: 'table' as const }, + { schema: 'public', name: 'orders', kind: 'table' as const }, + ]); + + const result = await runKtxSetupDatabasesStep( + { projectDir: tempDir, inputMode: 'auto', skipDatabases: false, databaseSchemas: [] }, + makeIo().io, + { prompts, testConnection, scanConnection, listSchemas, listTables }, + ); + + expect(result).toEqual({ status: 'ready', projectDir: tempDir, connectionIds: ['warehouse'] }); + expect(primaryMenuCount).toBe(2); + expect(listTables).toHaveBeenCalledWith(tempDir, 'warehouse'); + expect(scanConnection).not.toHaveBeenCalled(); + const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + expect(config.connections.warehouse).toMatchObject({ + url: 'env:DATABASE_URL', + schemas: ['public'], + enabled_tables: ['public.orders'], + }); + }); + + it('restores an existing database edit when the follow-up scan fails', async () => { + await writeFile( + join(tempDir, 'ktx.yaml'), + [ + 'project: warehouse', + 'connections:', + ' warehouse:', + ' driver: postgres', + ' url: env:DATABASE_URL', + ' schemas:', + ' - public', + ' enabled_tables:', + ' - public.orders', + 'setup:', + ' database_connection_ids:', + ' - warehouse', + '', + ].join('\n'), + 'utf-8', + ); + await writeKtxSetupState(tempDir, { completed_steps: ['databases'] }); + const prompts = makePromptAdapter({ + textValues: ['env:DATABASE_URL'], + multiselectValues: [['public']], + }); + vi.mocked(prompts.select).mockImplementation(async (options) => { + if (options.message === 'Databases already configured: warehouse\nWhat would you like to do?') return 'edit'; + if (options.message === 'Database to edit') return 'warehouse'; + if (options.message === 'How do you want to connect to PostgreSQL?') return 'url'; + if (options.message.startsWith('Enable query-history ingest')) return 'no'; + if (options.message.startsWith('Tables found in selected schemas')) return 'all'; + return 'back'; + }); + const listTables = vi.fn(async () => [ + { schema: 'public', name: 'customers', kind: 'table' as const }, + { schema: 'public', name: 'orders', kind: 'table' as const }, + ]); + + const result = await runKtxSetupDatabasesStep( + { projectDir: tempDir, inputMode: 'auto', skipDatabases: false, databaseSchemas: [] }, + makeIo().io, + { + prompts, + testConnection: vi.fn(async () => 0), + scanConnection: vi.fn(async () => 1), + listTables, + }, + ); + + expect(result).toEqual({ status: 'failed', projectDir: tempDir }); + const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); + expect(config.connections.warehouse).toMatchObject({ + enabled_tables: ['public.orders'], + }); + }); + it('lets Escape from connection fields return to connection method selection', async () => { const prompts = makePromptAdapter({ selectValues: ['fields', 'url'], @@ -887,7 +1277,6 @@ describe('setup databases step', () => { port: 5432, database: 'analytics', username: 'readonly', - readonly: true, }); expect(connection.password).toMatch(/^file:/); const secretPath = join(tempDir, '.ktx/secrets/postgres-warehouse-password'); @@ -941,7 +1330,7 @@ describe('setup databases step', () => { return 0; }); const scanConnection = vi.fn(async (_projectDir: string, _connectionId: string, commandIo: KtxCliIo) => { - commandIo.stdout.write('Scanning postgres-warehouse for context. Large primary sources can take a while.\n'); + commandIo.stdout.write('Scanning postgres-warehouse for context. Large databases can take a while.\n'); commandIo.stdout.write('[5%] Preparing scan\n'); commandIo.stdout.write('[15%] Inspecting database schema\n'); commandIo.stdout.write('[55%] Semantic layer comparison found 2 changes across 2 tables\n'); @@ -1038,7 +1427,6 @@ describe('setup databases step', () => { expect(config.connections['postgres-warehouse']).toMatchObject({ driver: 'postgres', url: 'env:DATABASE_URL', - readonly: true, }); }); @@ -1158,7 +1546,6 @@ describe('setup databases step', () => { url: 'env:DATABASE_URL', schemas: ['public'], context: { queryHistory: { enabled: false }, depth: 'fast' }, - readonly: true, }); expect(config.setup).toEqual({ database_connection_ids: ['warehouse'], @@ -1197,7 +1584,6 @@ describe('setup databases step', () => { expect(config.connections.warehouse).toEqual({ driver: 'sqlite', path: './warehouse.sqlite', - readonly: true, context: { depth: 'fast' }, }); expect(config.setup).toEqual({ @@ -1215,7 +1601,6 @@ describe('setup databases step', () => { ' warehouse:', ' driver: postgres', ' url: env:DATABASE_URL', - ' readonly: true', ' analytics:', ' driver: snowflake', ' authMethod: password', @@ -1225,7 +1610,6 @@ describe('setup databases step', () => { ' schema_name: PUBLIC', ' username: reader', ' password: env:SNOWFLAKE_PASSWORD', - ' readonly: true', '', ].join('\n'), 'utf-8', @@ -1591,7 +1975,6 @@ describe('setup databases step', () => { ' driver: bigquery', ' dataset_id: analytics', ' credentials_json: env:BIGQUERY_CREDENTIALS_JSON', - ' readonly: true', '', ].join('\n'), 'utf-8', @@ -1646,7 +2029,6 @@ describe('setup databases step', () => { ' warehouse:', ' driver: postgres', ' url: env:DATABASE_URL', - ' readonly: true', '', ].join('\n'), 'utf-8', diff --git a/packages/cli/src/setup-databases.ts b/packages/cli/src/setup-databases.ts index c34dc9f3..561ac455 100644 --- a/packages/cli/src/setup-databases.ts +++ b/packages/cli/src/setup-databases.ts @@ -177,6 +177,7 @@ const SCOPE_DISCOVERY_SPECS: Partial; +type ConnectionSetupStatus = 'ready' | 'back' | 'failed'; const DRIVER_CONNECTION_DEFAULTS: Record = { postgres: { port: '5432' }, @@ -234,6 +235,16 @@ function assertSafeDatabaseConnectionId(connectionId: string): void { } } +function stringConfigField(connection: KtxProjectConnectionConfig | undefined, field: string): string | undefined { + const value = connection?.[field]; + return typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined; +} + +function numberConfigField(connection: KtxProjectConnectionConfig | undefined, field: string): number | undefined { + const value = connection?.[field]; + return typeof value === 'number' && Number.isFinite(value) ? value : undefined; +} + function historicSqlConfigRecord(connection: KtxProjectConnectionConfig | undefined): Record | null { const historicSql = connection?.historicSql; return historicSql && typeof historicSql === 'object' && !Array.isArray(historicSql) @@ -503,6 +514,18 @@ function configuredPrimaryConnectionIds( .sort((left, right) => left.localeCompare(right)); } +function configuredPrimaryDrivers( + connections: Record, + connectionIds: string[], +): KtxSetupDatabaseDriver[] { + const configured = new Set( + connectionIds + .map((connectionId) => normalizeDriver(connections[connectionId]?.driver)) + .filter((driver): driver is KtxSetupDatabaseDriver => driver !== null), + ); + return DRIVER_OPTIONS.map((option) => option.value).filter((driver) => configured.has(driver)); +} + function configuredPrimarySourcesPrompt(connectionIds: string[]): { message: string; options: Array<{ value: string; label: string }>; @@ -511,6 +534,7 @@ function configuredPrimarySourcesPrompt(connectionIds: string[]): { message: `Databases already configured: ${connectionIds.join(', ')}\nWhat would you like to do?`, options: [ { value: 'continue', label: 'Continue to context sources' }, + { value: 'edit', label: 'Edit an existing database' }, { value: 'add', label: 'Add another database' }, ], }; @@ -601,23 +625,40 @@ async function buildFieldsConnectionConfig(input: { connectionId: string; args: KtxSetupDatabasesArgs; prompts: KtxSetupDatabasesPromptAdapter; + existingConnection?: KtxProjectConnectionConfig; }): Promise { const label = driverLabel(input.driver); const defaults = DRIVER_CONNECTION_DEFAULTS[input.driver]; - const host = await promptText(input.prompts, `${label} host`, 'localhost'); + const host = await promptText( + input.prompts, + `${label} host`, + stringConfigField(input.existingConnection, 'host') ?? 'localhost', + ); if (host === undefined) return 'back'; if (!host) return null; - const portStr = await promptText(input.prompts, `${label} port`, defaults.port); + const portStr = await promptText( + input.prompts, + `${label} port`, + String(numberConfigField(input.existingConnection, 'port') ?? defaults.port), + ); if (portStr === undefined) return 'back'; const port = Number(portStr || defaults.port); - const database = await promptText(input.prompts, `${label} database name`); + const database = await promptText( + input.prompts, + `${label} database name`, + stringConfigField(input.existingConnection, 'database'), + ); if (database === undefined) return 'back'; if (!database) return null; - const username = await promptText(input.prompts, `${label} username`); + const username = await promptText( + input.prompts, + `${label} username`, + stringConfigField(input.existingConnection, 'username'), + ); if (username === undefined) return 'back'; if (!username) return null; @@ -632,6 +673,7 @@ async function buildFieldsConnectionConfig(input: { }); if (credentialResult === 'back') return 'back'; if (credentialResult) passwordRef = credentialResult; + if (!credentialResult) passwordRef = stringConfigField(input.existingConnection, 'password'); } return { @@ -642,7 +684,6 @@ async function buildFieldsConnectionConfig(input: { username, ...(passwordRef ? { password: passwordRef } : {}), ...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}), - readonly: true, }; } @@ -651,9 +692,14 @@ async function buildPastedUrlConnectionConfig(input: { connectionId: string; args: KtxSetupDatabasesArgs; prompts: KtxSetupDatabasesPromptAdapter; + existingConnection?: KtxProjectConnectionConfig; }): Promise { const label = driverLabel(input.driver); - const rawUrl = await promptText(input.prompts, `${label} connection URL`); + const rawUrl = await promptText( + input.prompts, + `${label} connection URL`, + stringConfigField(input.existingConnection, 'url'), + ); if (rawUrl === undefined) return 'back'; if (!rawUrl) return null; @@ -664,7 +710,6 @@ async function buildPastedUrlConnectionConfig(input: { driver: input.driver, url, ...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}), - readonly: true, }; } @@ -678,7 +723,6 @@ async function buildPastedUrlConnectionConfig(input: { driver: input.driver, url: ref, ...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}), - readonly: true, }; } @@ -686,7 +730,6 @@ async function buildPastedUrlConnectionConfig(input: { driver: input.driver, url, ...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}), - readonly: true, }; } @@ -695,6 +738,7 @@ async function buildUrlConnectionConfig(input: { connectionId: string; args: KtxSetupDatabasesArgs; prompts: KtxSetupDatabasesPromptAdapter; + existingConnection?: KtxProjectConnectionConfig; }): Promise { if (input.args.inputMode === 'disabled' && !input.args.databaseUrl) return null; @@ -710,14 +754,12 @@ async function buildUrlConnectionConfig(input: { driver: input.driver, url: ref, ...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}), - readonly: true, }; } return { driver: input.driver, url, ...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}), - readonly: true, }; } @@ -744,6 +786,7 @@ async function buildConnectionConfig(input: { connectionId: string; args: KtxSetupDatabasesArgs; prompts: KtxSetupDatabasesPromptAdapter; + existingConnection?: KtxProjectConnectionConfig; }): Promise { const { driver, args, prompts } = input; if (driver === 'sqlite') { @@ -753,22 +796,37 @@ async function buildConnectionConfig(input: { (await promptText( prompts, 'SQLite database file\nEnter a relative or absolute path, for example ./warehouse.sqlite.', + stringConfigField(input.existingConnection, 'path'), )); if (path === undefined) return 'back'; - return path ? { driver: 'sqlite', path, readonly: true } : null; + return path ? { driver: 'sqlite', path } : null; } if (driver === 'postgres' || driver === 'mysql' || driver === 'clickhouse' || driver === 'sqlserver') { - return await buildUrlConnectionConfig({ driver, connectionId: input.connectionId, args, prompts }); + return await buildUrlConnectionConfig({ + driver, + connectionId: input.connectionId, + args, + prompts, + existingConnection: input.existingConnection, + }); } if (driver === 'bigquery') { - const datasetId = await promptText(prompts, 'BigQuery dataset\nFor example analytics.'); + const datasetId = await promptText( + prompts, + 'BigQuery dataset\nFor example analytics.', + stringConfigField(input.existingConnection, 'dataset_id'), + ); if (datasetId === undefined) return 'back'; - const credentialsPath = await promptText(prompts, 'Path to service account JSON file'); + const credentialsPath = await promptText( + prompts, + 'Path to service account JSON file', + stringConfigField(input.existingConnection, 'credentials_json'), + ); if (credentialsPath === undefined) return 'back'; const location = await promptText( prompts, 'BigQuery location\nPress Enter for US, or enter a location like EU.', - 'US', + stringConfigField(input.existingConnection, 'location') ?? 'US', ); if (location === undefined) return 'back'; if (!datasetId || !credentialsPath) return null; @@ -777,23 +835,38 @@ async function buildConnectionConfig(input: { dataset_id: datasetId, credentials_json: normalizeFileReference(credentialsPath), ...(location ? { location } : {}), - readonly: true, }; } if (driver === 'snowflake') { - const account = await promptText(prompts, 'Snowflake account identifier'); + const account = await promptText( + prompts, + 'Snowflake account identifier', + stringConfigField(input.existingConnection, 'account'), + ); if (account === undefined) return 'back'; - const warehouse = await promptText(prompts, 'Snowflake warehouse\nFor example ANALYTICS_WH.'); + const warehouse = await promptText( + prompts, + 'Snowflake warehouse\nFor example ANALYTICS_WH.', + stringConfigField(input.existingConnection, 'warehouse'), + ); if (warehouse === undefined) return 'back'; - const database = await promptText(prompts, 'Snowflake database name'); + const database = await promptText( + prompts, + 'Snowflake database name', + stringConfigField(input.existingConnection, 'database'), + ); if (database === undefined) return 'back'; const schemaName = await promptText( prompts, 'Snowflake schema\nPress Enter for PUBLIC, or enter a schema name.', - 'PUBLIC', + stringConfigField(input.existingConnection, 'schema_name') ?? 'PUBLIC', ); if (schemaName === undefined) return 'back'; - const username = await promptText(prompts, 'Snowflake username'); + const username = await promptText( + prompts, + 'Snowflake username', + stringConfigField(input.existingConnection, 'username'), + ); if (username === undefined) return 'back'; const passwordRef = await promptCredential({ prompts, @@ -803,9 +876,14 @@ async function buildConnectionConfig(input: { secretName: 'password', // pragma: allowlist secret }); if (passwordRef === 'back') return 'back'; // pragma: allowlist secret - const role = await promptText(prompts, 'Snowflake role (optional)\nPress Enter to skip.'); + const role = await promptText( + prompts, + 'Snowflake role (optional)\nPress Enter to skip.', + stringConfigField(input.existingConnection, 'role'), + ); if (role === undefined) return 'back'; - if (!account || !warehouse || !database || !schemaName || !username || !passwordRef) return null; + const resolvedPasswordRef = passwordRef ?? stringConfigField(input.existingConnection, 'password'); + if (!account || !warehouse || !database || !schemaName || !username || !resolvedPasswordRef) return null; return { driver: 'snowflake', authMethod: 'password', @@ -814,9 +892,8 @@ async function buildConnectionConfig(input: { database, schema_name: schemaName, username, - password: passwordRef, + password: resolvedPasswordRef, ...(role ? { role } : {}), - readonly: true, }; } throw new Error(`Unsupported database driver: ${driver}`); @@ -1134,6 +1211,59 @@ async function writeConnectionConfig(input: { } } +async function createConnectionConfigRollback(projectDir: string, connectionId: string): Promise<() => Promise> { + const project = await loadKtxProject({ projectDir }); + const previousConnection = project.config.connections[connectionId]; + const hadPreviousConnection = previousConnection !== undefined; + return async () => { + const latest = await loadKtxProject({ projectDir }); + const connections = { ...latest.config.connections }; + if (hadPreviousConnection) { + connections[connectionId] = previousConnection; + } else { + delete connections[connectionId]; + } + await writeFile( + latest.configPath, + serializeKtxProjectConfig({ + ...latest.config, + connections, + }), + 'utf-8', + ); + }; +} + +function withExistingPrimaryEditPromptDefaults(input: { + previous: KtxProjectConnectionConfig; + next: KtxProjectConnectionConfig; + driver: KtxSetupDatabaseDriver; +}): KtxProjectConnectionConfig { + const merged: KtxProjectConnectionConfig = { ...input.next }; + const spec = SCOPE_DISCOVERY_SPECS[input.driver]; + if (spec) { + const nextArray = input.next[spec.configArrayField]; + const previousArray = input.previous[spec.configArrayField]; + if ( + !(Array.isArray(nextArray) && nextArray.length > 0) && + Array.isArray(previousArray) && + previousArray.length > 0 + ) { + delete merged[spec.configSingleField]; + merged[spec.configArrayField] = previousArray; + } else if (!Object.hasOwn(input.next, spec.configArrayField) && !Object.hasOwn(input.next, spec.configSingleField)) { + const previousSingle = input.previous[spec.configSingleField]; + if (typeof previousSingle === 'string' && previousSingle.trim().length > 0) { + merged[spec.configSingleField] = previousSingle; + } + } + } + if (!Object.hasOwn(input.next, 'enabled_tables') && Array.isArray(input.previous.enabled_tables)) { + merged.enabled_tables = input.previous.enabled_tables; + } + return merged; +} + function configuredScopeValues( connection: KtxProjectConnectionConfig | undefined, spec: ScopeDiscoverySpec, @@ -1194,18 +1324,19 @@ async function maybeConfigureSchemaScope(input: { prompts: KtxSetupDatabasesPromptAdapter; deps: KtxSetupDatabasesDeps; io: KtxCliIo; -}): Promise { + forcePrompt?: boolean; +}): Promise { const project = await loadKtxProject({ projectDir: input.projectDir }); const connection = project.config.connections[input.connectionId]; const driver = normalizeDriver(connection?.driver); - if (!driver) return true; + if (!driver) return 'ready'; const spec = SCOPE_DISCOVERY_SPECS[driver]; - if (!spec) return true; + if (!spec) return 'ready'; const arrayVal = connection?.[spec.configArrayField]; - if (Array.isArray(arrayVal) && arrayVal.length > 0) { - return true; + if (Array.isArray(arrayVal) && arrayVal.length > 0 && input.forcePrompt !== true) { + return 'ready'; } if (input.args.databaseSchemas.length > 0) { @@ -1215,7 +1346,7 @@ async function maybeConfigureSchemaScope(input: { values: input.args.databaseSchemas, spec, }); - return true; + return 'ready'; } writeSetupSection(input.io, `Discovering ${spec.promptLabel.toLowerCase()}`, [ @@ -1228,14 +1359,18 @@ async function maybeConfigureSchemaScope(input: { await (input.deps.listSchemas ?? defaultListSchemas)(input.projectDir, input.connectionId), ); } catch (error) { + const detail = error instanceof Error ? error.message : String(error); input.io.stderr.write( - `Could not discover ${spec.promptLabel.toLowerCase()} for ${input.connectionId}; continuing with existing ${spec.noun} scope. ` + - `Pass --database-schema to set it explicitly. ${error instanceof Error ? error.message : String(error)}\n`, + input.forcePrompt === true + ? `Could not discover ${spec.promptLabel.toLowerCase()} for ${input.connectionId}; edit was not saved. ` + + `Pass --database-schema to set it explicitly. ${detail}\n` + : `Could not discover ${spec.promptLabel.toLowerCase()} for ${input.connectionId}; continuing with existing ${spec.noun} scope. ` + + `Pass --database-schema to set it explicitly. ${detail}\n`, ); - return true; + return input.forcePrompt === true ? 'failed' : 'ready'; } if (discovered.length === 0) { - return true; + return 'ready'; } let selected: string[]; @@ -1255,7 +1390,7 @@ async function maybeConfigureSchemaScope(input: { required: true, }); if (choices.includes('back')) { - return false; + return 'back'; } selected = choices.length > 0 ? choices : initialValues; } @@ -1270,7 +1405,7 @@ async function maybeConfigureSchemaScope(input: { writeSetupSection(input.io, `${capitalNounPlural} saved for ${input.connectionId}`, [ `✓ ${selected.join(', ')}`, ]); - return true; + return 'ready'; } async function maybeConfigureTableScope(input: { @@ -1280,19 +1415,20 @@ async function maybeConfigureTableScope(input: { prompts: KtxSetupDatabasesPromptAdapter; io: KtxCliIo; deps: KtxSetupDatabasesDeps; -}): Promise { + forcePrompt?: boolean; +}): Promise { const project = await loadKtxProject({ projectDir: input.projectDir }); const connection = project.config.connections[input.connectionId]; const driver = normalizeDriver(connection?.driver); - if (!driver || driver === 'sqlite') return true; + if (!driver || driver === 'sqlite') return 'ready'; const existingTables = connection?.enabled_tables; - if (Array.isArray(existingTables) && existingTables.length > 0) { - return true; + if (Array.isArray(existingTables) && existingTables.length > 0 && input.forcePrompt !== true) { + return 'ready'; } if (input.args.inputMode === 'disabled') { - return true; + return 'ready'; } writeSetupSection(input.io, 'Discovering tables', [ @@ -1306,15 +1442,20 @@ async function maybeConfigureTableScope(input: { input.connectionId, ); } catch (error) { + const detail = error instanceof Error ? error.message : String(error); input.io.stderr.write( - `Could not discover tables for ${input.connectionId}; continuing without table filter. ` + - `${error instanceof Error ? error.message : String(error)}\n`, + input.forcePrompt === true + ? `Could not discover tables for ${input.connectionId}; edit was not saved. ${detail}\n` + : `Could not discover tables for ${input.connectionId}; continuing without table filter. ${detail}\n`, ); - return true; + return input.forcePrompt === true ? 'failed' : 'ready'; } if (discovered.length === 0) { - return true; + if (input.forcePrompt === true) { + input.io.stderr.write(`No tables discovered for ${input.connectionId}; edit was not saved.\n`); + } + return input.forcePrompt === true ? 'failed' : 'ready'; } const allQualified = discovered.map((t) => `${t.schema}.${t.name}`); @@ -1328,7 +1469,7 @@ async function maybeConfigureTableScope(input: { writeSetupSection(input.io, `Tables enabled for ${input.connectionId}`, [ `✓ ${allQualified[0]}`, ]); - return true; + return 'ready'; } const bySchema = new Map(); @@ -1354,7 +1495,7 @@ async function maybeConfigureTableScope(input: { }); if (action === 'back') { - return false; + return 'back'; } if (action === 'all') { @@ -1370,7 +1511,10 @@ async function maybeConfigureTableScope(input: { const suffix = t.kind === 'view' ? ' (view)' : ''; return { value: qualified, label: `${qualified}${suffix}` }; }), - initialValues: allQualified, + initialValues: + Array.isArray(existingTables) && input.forcePrompt === true + ? existingTables.filter((table): table is string => typeof table === 'string' && allQualified.includes(table)) + : allQualified, required: true, }); @@ -1394,7 +1538,7 @@ async function maybeConfigureTableScope(input: { writeSetupSection(input.io, `Tables enabled for ${input.connectionId}`, [ `✓ ${selected.length}/${discovered.length} tables enabled`, ]); - return true; + return 'ready'; } async function ensureHistoricSqlIngestDefaults(projectDir: string): Promise { @@ -1546,7 +1690,8 @@ async function validateAndScanConnection(input: { deps: KtxSetupDatabasesDeps; args: KtxSetupDatabasesArgs; prompts: KtxSetupDatabasesPromptAdapter; -}): Promise { + forceScopeAndTables?: boolean; +}): Promise { const testConnection = input.deps.testConnection ?? defaultTestConnection; const scanConnection = input.deps.scanConnection ?? defaultScanConnection; const project = await loadKtxProject({ projectDir: input.projectDir }); @@ -1557,7 +1702,7 @@ async function validateAndScanConnection(input: { if (testCode !== 0) { flushBufferedCommandOutput(input.io, testIo); input.io.stderr.write(`Connection test failed for ${input.connectionId}.\n`); - return false; + return 'failed'; } const testOutput = testIo.stdoutText(); const outputDriver = normalizeDriver(readOutputValue(testOutput, 'Driver')); @@ -1566,14 +1711,24 @@ async function validateAndScanConnection(input: { writeSetupSection(input.io, `Testing ${input.connectionId}`, testLines); while (true) { - if (!(await maybeConfigureSchemaScope(input))) { - return false; + const schemaStatus = await maybeConfigureSchemaScope({ ...input, forcePrompt: input.forceScopeAndTables }); + if (schemaStatus !== 'ready') { + return schemaStatus; } - if (await maybeConfigureTableScope(input)) { + const tableStatus = await maybeConfigureTableScope({ ...input, forcePrompt: input.forceScopeAndTables }); + if (tableStatus === 'ready') { break; } + if (input.forceScopeAndTables) { + return tableStatus; + } + + if (tableStatus === 'failed') { + return 'failed'; + } + await clearScopeConfig(input.projectDir, input.connectionId); } @@ -1634,7 +1789,7 @@ async function validateAndScanConnection(input: { ); } if (scanCode !== 0) { - return false; + return 'failed'; } } const scanOutput = scanIo.stdoutText(); @@ -1646,14 +1801,14 @@ async function validateAndScanConnection(input: { writeSetupSection(input.io, 'Database ready', [ `${input.connectionId} · ${driverDisplay} · schema context complete`, ]); - return true; + return 'ready'; } async function chooseDrivers( args: KtxSetupDatabasesArgs, io: KtxCliIo, prompts: KtxSetupDatabasesPromptAdapter, - options?: { hasPrimarySources?: boolean }, + options?: { hasPrimarySources?: boolean; initialDrivers?: KtxSetupDatabaseDriver[] }, ): Promise { if (args.databaseDrivers && args.databaseDrivers.length > 0) { return [...new Set(args.databaseDrivers)]; @@ -1668,10 +1823,12 @@ async function chooseDrivers( return 'missing-input'; } while (true) { + const initialValues = unique(options?.initialDrivers ?? []); const choices = await prompts.multiselect({ message: withMultiselectNavigation('Which databases should KTX connect to?'), options: [...DRIVER_OPTIONS], - required: false, + ...(initialValues.length > 0 ? { initialValues } : {}), + required: options?.hasPrimarySources === true, }); if (choices.includes('back')) { return 'back'; @@ -1693,7 +1850,7 @@ async function chooseConnectionIdForDriver(input: { connections: Record; args: KtxSetupDatabasesArgs; prompts: KtxSetupDatabasesPromptAdapter; -}): Promise<{ kind: 'existing' | 'new'; connectionId: string } | 'back' | 'missing-input'> { +}): Promise<{ kind: 'existing' | 'new' | 'edit'; connectionId: string } | 'back' | 'missing-input'> { if (input.args.databaseConnectionId) { assertSafeDatabaseConnectionId(input.args.databaseConnectionId); return { kind: 'new', connectionId: input.args.databaseConnectionId }; @@ -1726,14 +1883,19 @@ async function chooseConnectionIdForDriver(input: { options: [ ...existingIds.map((connectionId) => ({ value: `existing:${connectionId}`, - label: `Use existing ${label} connection: ${connectionId}`, + label: `Keep existing ${label} connection: ${connectionId}`, })), - { value: 'new', label: `Add new ${label} connection` }, + ...existingIds.map((connectionId) => ({ + value: `edit:${connectionId}`, + label: `Edit ${label} connection: ${connectionId}`, + })), + { value: 'new', label: `Add another ${label} connection` }, { value: 'back', label: 'Back' }, ], }); if (choice === 'back') return 'back'; if (choice.startsWith('existing:')) return { kind: 'existing', connectionId: choice.slice('existing:'.length) }; + if (choice.startsWith('edit:')) return { kind: 'edit', connectionId: choice.slice('edit:'.length) }; const entered = await input.prompts.text({ message: withTextInputNavigation(connectionNamePrompt(label)), placeholder: defaultId, @@ -1746,6 +1908,102 @@ async function chooseConnectionIdForDriver(input: { } } +async function choosePrimarySourceToEdit(input: { + projectDir: string; + connectionIds: string[]; + prompts: KtxSetupDatabasesPromptAdapter; +}): Promise { + const project = await loadKtxProject({ projectDir: input.projectDir }); + const options = input.connectionIds + .map((connectionId) => { + const driver = normalizeDriver(project.config.connections[connectionId]?.driver); + if (!driver) return null; + return { value: connectionId, label: `${connectionId} (${driverLabel(driver)})` }; + }) + .filter((option): option is { value: string; label: string } => option !== null); + if (options.length === 0) return 'back'; + const choice = await input.prompts.select({ + message: 'Database to edit', + options: [...options, { value: 'back', label: 'Back' }], + }); + return choice === 'back' ? 'back' : choice; +} + +async function runPrimarySourceFullEdit(input: { + projectDir: string; + connectionId: string; + args: KtxSetupDatabasesArgs; + prompts: KtxSetupDatabasesPromptAdapter; + io: KtxCliIo; + deps: KtxSetupDatabasesDeps; +}): Promise<'ready' | 'back' | 'failed'> { + const project = await loadKtxProject({ projectDir: input.projectDir }); + const existing = project.config.connections[input.connectionId]; + const driver = normalizeDriver(existing?.driver); + if (!existing || !driver) { + input.io.stderr.write(`Connection "${input.connectionId}" is not a configured database.\n`); + return 'failed'; + } + + const rollback = await createConnectionConfigRollback(input.projectDir, input.connectionId); + const replacement = await buildConnectionConfig({ + driver, + connectionId: input.connectionId, + args: input.args, + prompts: input.prompts, + existingConnection: existing, + }); + if (replacement === 'back') { + await rollback(); + return 'back'; + } + if (!replacement) { + await rollback(); + return 'failed'; + } + + const withHistoricSql = await maybeApplyHistoricSqlConfig({ + connection: replacement, + driver, + args: input.args, + prompts: input.prompts, + }); + if (withHistoricSql === 'back') { + await rollback(); + return 'back'; + } + + await writeConnectionConfig({ + projectDir: input.projectDir, + connectionId: input.connectionId, + connection: withExistingPrimaryEditPromptDefaults({ + previous: existing, + next: { + ...withHistoricSql, + ...(!Object.hasOwn(withHistoricSql, 'historicSql') && existing.historicSql !== undefined + ? { historicSql: existing.historicSql } + : {}), + }, + driver, + }), + }); + + const validated = await validateAndScanConnection({ + projectDir: input.projectDir, + connectionId: input.connectionId, + io: input.io, + deps: input.deps, + args: input.args, + prompts: input.prompts, + forceScopeAndTables: true, + }); + if (validated !== 'ready') { + await rollback(); + return validated; + } + return 'ready'; +} + export async function runKtxSetupDatabasesStep( args: KtxSetupDatabasesArgs, io: KtxCliIo, @@ -1768,7 +2026,18 @@ export async function runKtxSetupDatabasesStep( prompts, }); if (historicSqlResult === 'back') return { status: 'back', projectDir: args.projectDir }; - if (!(await validateAndScanConnection({ projectDir: args.projectDir, connectionId, io, deps, args, prompts }))) { + const setupStatus = await validateAndScanConnection({ + projectDir: args.projectDir, + connectionId, + io, + deps, + args, + prompts, + }); + if (setupStatus === 'back') { + return { status: 'back', projectDir: args.projectDir }; + } + if (setupStatus === 'failed') { return { status: 'failed', projectDir: args.projectDir }; } selectedConnectionIds.push(connectionId); @@ -1792,10 +2061,43 @@ export async function runKtxSetupDatabasesStep( await markDatabasesComplete(args.projectDir, selectedConnectionIds); return { status: 'ready', projectDir: args.projectDir, connectionIds: selectedConnectionIds }; } + if (action === 'edit') { + const connectionId = await choosePrimarySourceToEdit({ + projectDir: args.projectDir, + connectionIds: selectedConnectionIds, + prompts, + }); + if (connectionId === 'back') { + showConfiguredPrimaryMenu = true; + continue; + } + const editResult = await runPrimarySourceFullEdit({ + projectDir: args.projectDir, + connectionId, + args, + prompts, + io, + deps, + }); + if (editResult === 'back') { + showConfiguredPrimaryMenu = true; + continue; + } + if (editResult === 'failed') { + return { status: 'failed', projectDir: args.projectDir }; + } + pushUniqueConnectionId(selectedConnectionIds, connectionId); + showConfiguredPrimaryMenu = true; + continue; + } } showConfiguredPrimaryMenu = false; - const drivers = await chooseDrivers(args, io, prompts, { hasPrimarySources: selectedConnectionIds.length > 0 }); + const driverProject = await loadKtxProject({ projectDir: args.projectDir }); + const drivers = await chooseDrivers(args, io, prompts, { + hasPrimarySources: selectedConnectionIds.length > 0, + initialDrivers: configuredPrimaryDrivers(driverProject.config.connections, selectedConnectionIds), + }); if (drivers === 'back') { if (selectedConnectionIds.length > 0 && canReturnToDriverSelection && args.inputMode !== 'disabled') { showConfiguredPrimaryMenu = true; @@ -1836,7 +2138,26 @@ export async function runKtxSetupDatabasesStep( return { status: 'missing-input', projectDir: args.projectDir }; } - if (connectionChoice.kind === 'new') { + let connectionAlreadyValidated = false; + if (connectionChoice.kind === 'edit') { + const editResult = await runPrimarySourceFullEdit({ + projectDir: args.projectDir, + connectionId: connectionChoice.connectionId, + args, + prompts, + io, + deps, + }); + if (editResult === 'back') { + if (!canReturnToDriverSelection) return { status: 'back', projectDir: args.projectDir }; + returnToDriverSelection = true; + break; + } + if (editResult === 'failed') { + return { status: 'failed', projectDir: args.projectDir }; + } + connectionAlreadyValidated = true; + } else if (connectionChoice.kind === 'new') { let connection = await buildConnectionConfig({ driver, connectionId: connectionChoice.connectionId, @@ -1929,16 +2250,22 @@ export async function runKtxSetupDatabasesStep( } let connectionSkipped = false; - while ( - !(await validateAndScanConnection({ - projectDir: args.projectDir, - connectionId: connectionChoice.connectionId, - io, - deps, - args, - prompts, - })) - ) { + let setupStatus: ConnectionSetupStatus = connectionAlreadyValidated + ? 'ready' + : await validateAndScanConnection({ + projectDir: args.projectDir, + connectionId: connectionChoice.connectionId, + io, + deps, + args, + prompts, + }); + while (!connectionAlreadyValidated && setupStatus !== 'ready') { + if (setupStatus === 'back') { + if (!canReturnToDriverSelection) return { status: 'back', projectDir: args.projectDir }; + returnToDriverSelection = true; + break; + } if (args.inputMode === 'disabled') return { status: 'failed', projectDir: args.projectDir }; const action = await prompts.select({ message: `Database setup failed for ${connectionChoice.connectionId}`, @@ -1958,7 +2285,16 @@ export async function runKtxSetupDatabasesStep( connectionSkipped = true; break; } - if (action === 're-enter') { + if (action === 'retry') { + setupStatus = await validateAndScanConnection({ + projectDir: args.projectDir, + connectionId: connectionChoice.connectionId, + io, + deps, + args, + prompts, + }); + } else if (action === 're-enter') { const connection = await buildConnectionConfig({ driver, connectionId: connectionChoice.connectionId, @@ -1994,6 +2330,14 @@ export async function runKtxSetupDatabasesStep( connectionId: connectionChoice.connectionId, connection: withContextDepth, }); + setupStatus = await validateAndScanConnection({ + projectDir: args.projectDir, + connectionId: connectionChoice.connectionId, + io, + deps, + args, + prompts, + }); } } if (returnToDriverSelection) break; diff --git a/packages/cli/src/setup-embeddings.test.ts b/packages/cli/src/setup-embeddings.test.ts index c2d5cad2..4c91ddd5 100644 --- a/packages/cli/src/setup-embeddings.test.ts +++ b/packages/cli/src/setup-embeddings.test.ts @@ -319,14 +319,14 @@ describe('setup embeddings step', () => { projectDir: tempDir, inputMode: 'disabled', embeddingBackend: 'openai', - embeddingApiKeyEnv: 'OPENAI_API_KEY', + embeddingApiKeyEnv: 'OPENAI_API_KEY', // pragma: allowlist secret cliVersion: '0.2.0', runtimeInstallPolicy: 'auto', skipEmbeddings: false, }, io.io, { - env: { OPENAI_API_KEY: 'sk-openai-test' }, + env: { OPENAI_API_KEY: 'sk-openai-test' }, // pragma: allowlist secret healthCheck, }, ); @@ -336,14 +336,14 @@ describe('setup embeddings step', () => { backend: 'openai', model: 'text-embedding-3-small', dimensions: 1536, - openai: { apiKey: 'sk-openai-test' }, + openai: { apiKey: 'sk-openai-test' }, // pragma: allowlist secret }); const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')); expect(config.ingest.embeddings).toMatchObject({ backend: 'openai', model: 'text-embedding-3-small', dimensions: 1536, - openai: { api_key: 'env:OPENAI_API_KEY' }, + openai: { api_key: 'env:OPENAI_API_KEY' }, // pragma: allowlist secret }); expect(io.stdout()).not.toContain('sk-openai-test'); }); @@ -367,7 +367,7 @@ describe('setup embeddings step', () => { io.io, { prompts, - env: { OPENAI_API_KEY: 'sk-openai-test' }, + env: { OPENAI_API_KEY: 'sk-openai-test' }, // pragma: allowlist secret healthCheck, ensureLocalEmbeddings: vi.fn(async () => managedDaemon()), }, @@ -384,7 +384,7 @@ describe('setup embeddings step', () => { backend: 'openai', model: 'text-embedding-3-small', dimensions: 1536, - openai: { apiKey: 'sk-openai-test' }, + openai: { apiKey: 'sk-openai-test' }, // pragma: allowlist secret }); expect(prompts.select).toHaveBeenCalledWith( expect.objectContaining({ @@ -478,7 +478,7 @@ describe('setup embeddings step', () => { }, makeIo().io, { - env: { OPENAI_API_KEY: 'sk-openai-test' }, + env: { OPENAI_API_KEY: 'sk-openai-test' }, // pragma: allowlist secret healthCheck, }, ), diff --git a/packages/cli/src/setup-models.test.ts b/packages/cli/src/setup-models.test.ts index e310ea90..e4425d69 100644 --- a/packages/cli/src/setup-models.test.ts +++ b/packages/cli/src/setup-models.test.ts @@ -7,10 +7,8 @@ import { BUNDLED_ANTHROPIC_MODELS, fetchAnthropicModels, type KtxSetupModelPromptAdapter, - runKtxSetupGcloudApplicationDefaultAuth, runKtxSetupAnthropicModelStep, } from './setup-models.js'; -import type { KtxCliIo } from './cli-runtime.js'; function makeIo() { let stdout = ''; @@ -34,6 +32,17 @@ function makeIo() { }; } +function makeSpinnerEvents() { + const events: string[] = []; + const spinner = vi.fn(() => ({ + start: (msg: string) => events.push(`start:${msg}`), + message: (msg: string) => events.push(`message:${msg}`), + stop: (msg: string) => events.push(`stop:${msg}`), + error: (msg: string) => events.push(`error:${msg}`), + })); + return { events, spinner }; +} + function makePromptAdapter(options: { providerChoice?: string; selectValues?: string[]; @@ -191,6 +200,7 @@ describe('setup Anthropic model step', () => { it('configures env credentials, selected model, prompt caching, and llm completion state', async () => { const io = makeIo(); + const { events: spinnerEvents, spinner } = makeSpinnerEvents(); const result = await runKtxSetupAnthropicModelStep( { projectDir: tempDir, @@ -203,6 +213,7 @@ describe('setup Anthropic model step', () => { { env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, // pragma: allowlist secret healthCheck: vi.fn(async () => ({ ok: true as const })), + spinner, }, ); @@ -219,6 +230,10 @@ describe('setup Anthropic model step', () => { expect(config.scan.enrichment.mode).toBe('llm'); expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:'); expect((await readKtxSetupState(tempDir)).completed_steps).toContain('llm'); + expect(spinnerEvents).toEqual([ + 'start:Checking Anthropic API LLM (claude-sonnet-4-6).', + 'stop:LLM test passed (Anthropic API, claude-sonnet-4-6)', + ]); expect(io.stdout()).toContain('LLM ready: yes'); expect(io.stdout()).not.toContain('sk-ant-test'); }); @@ -226,6 +241,7 @@ describe('setup Anthropic model step', () => { it('configures Vertex AI provider, selected model, prompt caching, and llm completion state', async () => { const io = makeIo(); const healthCheck = vi.fn(async () => ({ ok: true as const })); + const { events: spinnerEvents, spinner } = makeSpinnerEvents(); const result = await runKtxSetupAnthropicModelStep( { @@ -238,7 +254,7 @@ describe('setup Anthropic model step', () => { skipLlm: false, }, io.io, - { env: {}, healthCheck }, + { env: {}, healthCheck, spinner }, ); expect(result.status).toBe('ready'); @@ -260,13 +276,16 @@ describe('setup Anthropic model step', () => { expect(config.scan.enrichment.mode).toBe('llm'); expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:'); expect((await readKtxSetupState(tempDir)).completed_steps).toContain('llm'); + expect(spinnerEvents).toEqual([ + 'start:Checking Vertex AI LLM (claude-sonnet-4-6).', + 'stop:LLM test passed (Vertex AI, claude-sonnet-4-6)', + ]); expect(io.stdout()).toContain('LLM ready: yes (claude-sonnet-4-6)'); }); - it('can run gcloud auth for Vertex AI and infer project and default location', async () => { + it('uses existing Vertex AI credentials without offering to run gcloud auth', async () => { const io = makeIo(); - const prompts = makePromptAdapter({ selectValues: ['vertex', 'gcloud', 'local-gcp-project', 'claude-sonnet-4-6'] }); - const runGcloudAuth = vi.fn(async () => ({ ok: true as const })); + const prompts = makePromptAdapter({ selectValues: ['vertex', 'existing', 'local-gcp-project', 'claude-sonnet-4-6'] }); const readGcloudProject = vi.fn(async () => 'local-gcp-project'); const listGcloudProjects = vi.fn(async () => [ { projectId: 'local-gcp-project', name: 'Local project' }, @@ -280,7 +299,6 @@ describe('setup Anthropic model step', () => { { prompts, env: {}, - runGcloudAuth, readGcloudProject, listGcloudProjects, healthCheck, @@ -288,7 +306,15 @@ describe('setup Anthropic model step', () => { ); expect(result.status).toBe('ready'); - expect(runGcloudAuth).toHaveBeenCalledWith(io.io); + expect(prompts.select).toHaveBeenCalledWith( + expect.objectContaining({ + message: expect.stringContaining('How should KTX authenticate with Google Vertex AI?'), + options: [ + { value: 'existing', label: 'Use existing gcloud/Application Default Credentials' }, + { value: 'back', label: 'Back' }, + ], + }), + ); expect(readGcloudProject).toHaveBeenCalled(); expect(listGcloudProjects).toHaveBeenCalled(); expect(prompts.text).not.toHaveBeenCalled(); @@ -303,6 +329,22 @@ describe('setup Anthropic model step', () => { ], }), ); + expect(prompts.select).toHaveBeenCalledWith( + expect.objectContaining({ + message: expect.stringContaining('Which Anthropic model should KTX use?'), + options: [ + { value: 'claude-opus-4-7', label: 'Claude Opus 4.7' }, + { value: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6' }, + { value: 'claude-opus-4-6', label: 'Claude Opus 4.6' }, + { value: 'claude-opus-4-5', label: 'Claude Opus 4.5' }, + { value: 'claude-haiku-4-5', label: 'Claude Haiku 4.5' }, + { value: 'claude-sonnet-4-5', label: 'Claude Sonnet 4.5' }, + { value: 'claude-opus-4-1', label: 'Claude Opus 4.1' }, + { value: 'manual', label: 'Enter a model ID manually' }, + { value: 'back', label: 'Back' }, + ], + }), + ); expect(healthCheck).toHaveBeenCalledWith({ backend: 'vertex', vertex: { project: 'local-gcp-project', location: 'us-east5' }, @@ -415,35 +457,6 @@ describe('setup Anthropic model step', () => { ); }); - it('runs only gcloud application-default login for Vertex AI auth', async () => { - const io = makeIo(); - const runGcloud = vi.fn(async () => ({ ok: true as const })); - - await expect(runKtxSetupGcloudApplicationDefaultAuth(io.io, runGcloud)).resolves.toEqual({ ok: true }); - - expect(runGcloud).toHaveBeenCalledTimes(1); - expect(runGcloud).toHaveBeenCalledWith(['auth', 'application-default', 'login'], expect.anything()); - expect(runGcloud).not.toHaveBeenCalledWith(['auth', 'login'], expect.anything()); - expect(io.stdout()).toContain('gcloud auth application-default login'); - expect(io.stdout()).not.toContain('gcloud auth login'); - }); - - it('indents gcloud auth output inside the setup gutter', async () => { - const io = makeIo(); - const runGcloud = vi.fn(async (_args: string[], commandIo: KtxCliIo) => { - commandIo.stdout.write('Your browser has been opened to visit:\n\n https://accounts.example/auth\n'); - commandIo.stderr.write('Credentials saved to file: [/tmp/application_default_credentials.json]\n'); - return { ok: true as const }; - }); - - await expect(runKtxSetupGcloudApplicationDefaultAuth(io.io, runGcloud)).resolves.toEqual({ ok: true }); - - expect(io.stdout()).toContain('│ Your browser has been opened to visit:'); - expect(io.stdout()).toContain('│ https://accounts.example/auth'); - expect(io.stderr()).toContain('│ Credentials saved to file: [/tmp/application_default_credentials.json]'); - expect(io.stdout()).not.toContain('\nYour browser has been opened'); - }); - it('explains common Vertex AI Forbidden health-check causes', async () => { const io = makeIo(); diff --git a/packages/cli/src/setup-models.ts b/packages/cli/src/setup-models.ts index bd05bd44..e4c7fcd2 100644 --- a/packages/cli/src/setup-models.ts +++ b/packages/cli/src/setup-models.ts @@ -1,4 +1,4 @@ -import { execFile, spawn } from 'node:child_process'; +import { execFile } from 'node:child_process'; import { writeFile } from 'node:fs/promises'; import { promisify } from 'node:util'; import { resolveLocalKtxLlmConfig } from '@ktx/context'; @@ -11,6 +11,7 @@ import { serializeKtxProjectConfig, } from '@ktx/context/project'; import { type KtxLlmConfig, type KtxLlmHealthCheckResult, runKtxLlmHealthCheck } from '@ktx/llm'; +import { createClackSpinner, type KtxCliSpinner } from './clack.js'; import type { KtxCliIo } from './cli-runtime.js'; import { withTextInputNavigation } from './prompt-navigation.js'; import { envCredentialReference, writeProjectLocalSecretReference } from './setup-secrets.js'; @@ -61,9 +62,9 @@ export interface KtxSetupModelDeps { prompts?: KtxSetupModelPromptAdapter; listModels?: (apiKey: string) => Promise; healthCheck?: (config: KtxLlmConfig) => Promise; - runGcloudAuth?: (io: KtxCliIo) => Promise; readGcloudProject?: () => Promise; listGcloudProjects?: () => Promise; + spinner?: () => KtxCliSpinner; } export const BUNDLED_ANTHROPIC_MODEL_REGISTRY_VERSION = '2026-05-07'; @@ -74,6 +75,16 @@ export const BUNDLED_ANTHROPIC_MODELS: AnthropicModelChoice[] = [ { id: 'claude-haiku-4-5', label: 'Claude Haiku 4.5', recommended: false }, ]; +const VERTEX_ANTHROPIC_MODELS: AnthropicModelChoice[] = [ + { id: 'claude-opus-4-7', label: 'Claude Opus 4.7', recommended: false }, + { id: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6', recommended: false }, + { id: 'claude-opus-4-6', label: 'Claude Opus 4.6', recommended: false }, + { id: 'claude-opus-4-5', label: 'Claude Opus 4.5', recommended: false }, + { id: 'claude-haiku-4-5', label: 'Claude Haiku 4.5', recommended: false }, + { id: 'claude-sonnet-4-5', label: 'Claude Sonnet 4.5', recommended: false }, + { id: 'claude-opus-4-1', label: 'Claude Opus 4.1', recommended: false }, +]; + const HIDDEN_ANTHROPIC_MODEL_PATTERNS = [ /^claude-sonnet-4$/i, /^claude-opus-4$/i, @@ -91,8 +102,8 @@ const ANTHROPIC_MODEL_PROMPT_CONTEXT = 'into semantic-layer sources and wiki context.'; const VERTEX_AUTH_PROMPT_CONTEXT = - 'KTX can use Google Cloud Application Default Credentials for local Vertex AI access. This opens the normal ' + - 'gcloud browser login flow and does not store Google credentials in ktx.yaml.'; + 'KTX uses Google Cloud Application Default Credentials for local Vertex AI access and does not store Google ' + + 'credentials in ktx.yaml. If needed, run gcloud auth application-default login before continuing.'; const VERTEX_PROJECT_PROMPT_CONTEXT = 'KTX stores the selected Google Cloud project ID in ktx.yaml and uses Application Default Credentials for ' + 'access. Project visibility depends on the signed-in Google account and organization permissions.'; @@ -137,94 +148,17 @@ type VertexConfigChoice = } | { status: 'back' | 'missing-input' }; -type VertexAuthChoice = { status: 'ready' } | { status: 'back' | 'missing-input' }; +type VertexAuthChoice = { status: 'ready' } | { status: 'back' }; -export type GcloudAuthResult = { ok: true } | { ok: false; message: string }; interface GcloudProjectChoice { projectId: string; name?: string; } -type GcloudCommandRunner = (args: string[], io: KtxCliIo) => Promise; function createPromptAdapter(): KtxSetupModelPromptAdapter { return createKtxSetupPromptAdapter({ selectCancelValue: 'back' }); } -function createIndentedCommandIo(io: KtxCliIo): KtxCliIo { - const indentedWriter = (write: (chunk: string) => void) => { - let atLineStart = true; - return (chunk: string) => { - for (const char of chunk) { - if (atLineStart) { - write('│ '); - atLineStart = false; - } - write(char); - if (char === '\n') { - atLineStart = true; - } - } - }; - }; - - return { - stdout: { - isTTY: io.stdout.isTTY, - columns: io.stdout.columns, - write: indentedWriter((chunk) => io.stdout.write(chunk)), - }, - stderr: { - write: indentedWriter((chunk) => io.stderr.write(chunk)), - }, - }; -} - -function runInteractiveGcloud(args: string[], io: KtxCliIo): Promise { - return new Promise((resolve) => { - let settled = false; - const child = spawn('gcloud', args, { stdio: ['inherit', 'pipe', 'pipe'] }); - child.stdout?.on('data', (chunk: Buffer) => { - io.stdout.write(chunk.toString('utf8')); - }); - child.stderr?.on('data', (chunk: Buffer) => { - io.stderr.write(chunk.toString('utf8')); - }); - child.on('error', (error: NodeJS.ErrnoException) => { - if (settled) { - return; - } - settled = true; - if (error.code === 'ENOENT') { - resolve({ ok: false, message: 'gcloud CLI was not found on PATH.' }); - return; - } - resolve({ ok: false, message: error.message }); - }); - child.on('close', (code, signal) => { - if (settled) { - return; - } - settled = true; - if (code === 0) { - resolve({ ok: true }); - return; - } - resolve({ - ok: false, - message: signal ? `gcloud exited after signal ${signal}.` : `gcloud exited with code ${code ?? 'unknown'}.`, - }); - }); - }); -} - -export async function runKtxSetupGcloudApplicationDefaultAuth( - io: KtxCliIo, - runGcloud: GcloudCommandRunner = runInteractiveGcloud, -): Promise { - io.stdout.write('│ Running gcloud auth application-default login...\n'); - return await runGcloud(['auth', 'application-default', 'login'], createIndentedCommandIo(io)); -} - async function defaultReadGcloudProject(): Promise { try { const { stdout } = await execFileAsync('gcloud', ['config', 'get-value', 'project'], { encoding: 'utf8' }); @@ -374,6 +308,53 @@ function buildVertexHealthConfig(vertex: { project?: string; location: string }, }; } +type LlmHealthProvider = 'Anthropic API' | 'Vertex AI'; + +function llmHealthCheckStartText(provider: LlmHealthProvider, model: string): string { + return `Checking ${provider} LLM (${model}).`; +} + +function startLlmHealthCheckProgress( + spinner: KtxCliSpinner, + message: string, +): { succeed(msg: string): void; fail(msg: string): void } { + spinner.start(message); + return { + succeed(msg: string) { + spinner.stop(msg); + }, + fail(msg: string) { + spinner.error(msg); + }, + }; +} + +async function runLlmHealthCheckWithProgress( + config: KtxLlmConfig, + provider: LlmHealthProvider, + model: string, + healthCheck: (config: KtxLlmConfig) => Promise, + deps: KtxSetupModelDeps, +): Promise { + const progress = startLlmHealthCheckProgress( + (deps.spinner ?? createClackSpinner)(), + llmHealthCheckStartText(provider, model), + ); + let health: KtxLlmHealthCheckResult; + try { + health = await healthCheck(config); + } catch (error) { + progress.fail('LLM test failed'); + throw error; + } + if (health.ok) { + progress.succeed(`LLM test passed (${provider}, ${model})`); + } else { + progress.fail('LLM test failed'); + } + return health; +} + function formatVertexHealthFailure(message: string, vertex: { project?: string; location: string }): string { const trimmed = message.trim() || 'unknown error'; if (!/(forbidden|permission|permission_denied|403)/i.test(trimmed)) { @@ -516,7 +497,6 @@ async function chooseBackend( async function chooseVertexAuth( args: KtxSetupModelArgs, - io: KtxCliIo, deps: KtxSetupModelDeps, ): Promise { if (args.inputMode === 'disabled' || args.vertexProject || args.vertexLocation) { @@ -527,7 +507,6 @@ async function chooseVertexAuth( const choice = await prompts.select({ message: `How should KTX authenticate with Google Vertex AI?\n\n${VERTEX_AUTH_PROMPT_CONTEXT}`, options: [ - { value: 'gcloud', label: 'Run gcloud Application Default Credentials login' }, { value: 'existing', label: 'Use existing gcloud/Application Default Credentials' }, { value: 'back', label: 'Back' }, ], @@ -535,15 +514,6 @@ async function chooseVertexAuth( if (choice === 'back') { return { status: 'back' }; } - if (choice !== 'gcloud') { - return { status: 'ready' }; - } - - const result = await (deps.runGcloudAuth ?? runKtxSetupGcloudApplicationDefaultAuth)(io); - if (!result.ok) { - io.stderr.write(`gcloud authentication failed: ${result.message}\n`); - return { status: 'missing-input' }; - } return { status: 'ready' }; } @@ -799,7 +769,7 @@ async function chooseVertexModel(args: KtxSetupModelArgs, io: KtxCliIo, deps: Kt return { status: 'missing-input' }; } - const selectableModels = BUNDLED_ANTHROPIC_MODELS.filter(isSelectableAnthropicModel); + const selectableModels = VERTEX_ANTHROPIC_MODELS.filter(isSelectableAnthropicModel); const prompts = deps.prompts ?? createPromptAdapter(); const choice = await prompts.select({ message: `Which Anthropic model should KTX use?\n\n${ANTHROPIC_MODEL_PROMPT_CONTEXT}`, @@ -901,7 +871,7 @@ export async function runKtxSetupAnthropicModelStep( : attemptArgs; if (backendChoice.backend === 'vertex') { - const auth = await chooseVertexAuth(backendArgs, io, deps); + const auth = await chooseVertexAuth(backendArgs, deps); if (auth.status === 'back' && backendChoice.prompted) { attemptArgs = buildInteractiveRetryArgs(args); continue; @@ -931,7 +901,13 @@ export async function runKtxSetupAnthropicModelStep( return { status: model.status, projectDir: args.projectDir }; } - const health = await healthCheck(buildVertexHealthConfig(vertex.values, model.model)); + const health = await runLlmHealthCheckWithProgress( + buildVertexHealthConfig(vertex.values, model.model), + 'Vertex AI', + model.model, + healthCheck, + deps, + ); if (health.ok) { await persistLlmConfig(args.projectDir, { backend: 'vertex', vertex: vertex.refs }, model.model); io.stdout.write(`│ LLM ready: yes (${model.model})\n`); @@ -973,7 +949,13 @@ export async function runKtxSetupAnthropicModelStep( return { status: model.status, projectDir: args.projectDir }; } - const health = await healthCheck(buildAnthropicHealthConfig(credential.value, model.model)); + const health = await runLlmHealthCheckWithProgress( + buildAnthropicHealthConfig(credential.value, model.model), + 'Anthropic API', + model.model, + healthCheck, + deps, + ); if (health.ok) { await persistLlmConfig(args.projectDir, { backend: 'anthropic', credentialRef: credential.ref }, model.model); io.stdout.write(`│ LLM ready: yes (${model.model})\n`); diff --git a/packages/cli/src/setup-sources.test.ts b/packages/cli/src/setup-sources.test.ts index 319f653e..67c5e126 100644 --- a/packages/cli/src/setup-sources.test.ts +++ b/packages/cli/src/setup-sources.test.ts @@ -98,7 +98,7 @@ describe('setup sources step', () => { ...config, connections: { ...config.connections, - warehouse: { driver: 'postgres', url: 'env:DATABASE_URL', readonly: true }, + warehouse: { driver: 'postgres', url: 'env:DATABASE_URL' }, }, setup: { ...config.setup, @@ -486,7 +486,6 @@ describe('setup sources step', () => { driver: 'snowflake', account: 'acme', database: 'analytics', - readonly: true, }); const cases: Array<{ @@ -787,7 +786,7 @@ describe('setup sources step', () => { expect(testPrompts.text).toHaveBeenCalledTimes(4); }); - it('adds a dbt source connection without adapter allow-list entries', async () => { + it('adds a dbt source connection and enables its adapter', async () => { await addPrimarySource(); const validateDbt = vi.fn(async () => ({ ok: true as const, detail: 'project=analytics schemas=2' })); @@ -810,8 +809,7 @@ describe('setup sources step', () => { const configText = await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'); expect(configText).not.toContain('live-database'); expect(configText).not.toContain('historic-sql'); - expect(configText).not.toMatch(/^\s+adapters:/m); - expect((await readConfig()).ingest.adapters).toEqual([]); + expect((await readConfig()).ingest.adapters).toEqual(['dbt']); }); it('lets interactive setup retry or continue after initial source ingest fails', async () => { @@ -899,6 +897,7 @@ describe('setup sources step', () => { message: 'Configure dbt', options: [ { value: 'existing:dbt-main', label: 'Use existing dbt connection: dbt-main' }, + { value: 'edit:dbt-main', label: 'Edit existing dbt connection: dbt-main' }, { value: 'new', label: 'Add new dbt connection' }, { value: 'back', label: 'Back' }, ], @@ -1026,6 +1025,10 @@ describe('setup sources step', () => { value: `existing:${testCase.connectionId}`, label: `Use existing ${testCase.expectedLabel} connection: ${testCase.connectionId}`, }, + { + value: `edit:${testCase.connectionId}`, + label: `Edit existing ${testCase.expectedLabel} connection: ${testCase.connectionId}`, + }, { value: 'new', label: `Add new ${testCase.expectedLabel} connection` }, { value: 'back', label: 'Back' }, ], @@ -1034,6 +1037,314 @@ describe('setup sources step', () => { } }); + it('edits an existing Notion source and reopens the page picker with stored pages selected', async () => { + await addPrimarySource(); + await addConnection('notion-main', { + driver: 'notion', + auth_token_ref: 'env:NOTION_TOKEN', + crawl_mode: 'selected_roots', + root_page_ids: ['old-page'], + root_database_ids: [], + root_data_source_ids: [], + }); + const validateNotion = vi.fn(async () => ({ ok: true as const, detail: 'roots=1' })); + const pickNotionRootPages = vi.fn(async () => ({ kind: 'selected' as const, rootPageIds: ['new-page'] })); + const testPrompts = prompts({ + multiselect: [['notion']], + select: ['edit:notion-main', 'keep', 'selected_roots', 'done'], + }); + + await expect( + runKtxSetupSourcesStep( + { projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false }, + makeIo().io, + { + prompts: testPrompts, + validateNotion, + pickNotionRootPages, + }, + ), + ).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['notion-main'] }); + + expect(testPrompts.select).toHaveBeenCalledWith({ + message: 'How should KTX find your Notion integration token?', + options: [ + { value: 'keep', label: 'Keep existing credential' }, + { value: 'env', label: 'Use NOTION_TOKEN from the environment' }, + { value: 'paste', label: 'Paste a key and save it as a local secret file' }, + { value: 'back', label: 'Back' }, + ], + }); + expect(pickNotionRootPages).toHaveBeenCalledWith( + { + connectionId: 'notion-main', + connection: expect.objectContaining({ + driver: 'notion', + auth_token_ref: 'env:NOTION_TOKEN', + crawl_mode: 'selected_roots', + root_page_ids: ['old-page'], + }), + }, + expect.anything(), + ); + expect((await readConfig()).connections['notion-main']).toMatchObject({ + driver: 'notion', + auth_token_ref: 'env:NOTION_TOKEN', + crawl_mode: 'selected_roots', + root_page_ids: ['new-page'], + }); + }); + + it('edits an existing Metabase source with the current URL and credential as defaults', async () => { + await addPrimarySource(); + await addConnection('metabase-main', { + driver: 'metabase', + api_url: 'https://metabase-old.example.com', + api_key_ref: 'env:METABASE_API_KEY', // pragma: allowlist secret + mappings: { + databaseMappings: { '1': 'warehouse' }, + syncEnabled: { '1': true }, + syncMode: 'ALL', + }, + }); + const testPrompts = prompts({ + multiselect: [['metabase']], + select: ['edit:metabase-main', 'keep', 'done'], + text: ['https://metabase-new.example.com'], + }); + const discoverMetabaseDatabases = vi.fn(async () => [ + { id: 2, name: 'Analytics', engine: 'postgres', host: 'db.example.com', dbName: 'analytics' }, + ]); + + await expect( + runKtxSetupSourcesStep( + { projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false }, + makeIo().io, + { + prompts: testPrompts, + discoverMetabaseDatabases, + validateMetabase: vi.fn(async () => ({ ok: true as const, detail: 'mapping validated' })), + runMapping: vi.fn(async () => 0), + }, + ), + ).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['metabase-main'] }); + + expect(testPrompts.text).toHaveBeenCalledWith({ + message: textInputPrompt('Metabase URL'), + initialValue: 'https://metabase-old.example.com', + }); + expect(testPrompts.select).toHaveBeenCalledWith({ + message: 'How should KTX find your Metabase API key?', + options: [ + { value: 'keep', label: 'Keep existing credential' }, + { value: 'env', label: 'Use METABASE_API_KEY from the environment' }, + { value: 'paste', label: 'Paste a key and save it as a local secret file' }, + { value: 'back', label: 'Back' }, + ], + }); + expect(discoverMetabaseDatabases).toHaveBeenCalledWith({ + sourceUrl: 'https://metabase-new.example.com', + sourceApiKeyRef: 'env:METABASE_API_KEY', + sourceConnectionId: 'metabase-main', + }); + expect((await readConfig()).connections['metabase-main']).toMatchObject({ + driver: 'metabase', + api_url: 'https://metabase-new.example.com', + api_key_ref: 'env:METABASE_API_KEY', + mappings: { + databaseMappings: { '2': 'warehouse' }, + syncEnabled: { '2': true }, + syncMode: 'ALL', + }, + }); + }); + + it('rolls back an edited context source when validation fails', async () => { + await addPrimarySource(); + await addConnection('dbt-main', { + driver: 'dbt', + source_dir: '/repo/existing-dbt', + project_name: 'analytics', + }); + const validateDbt = vi.fn(async () => ({ ok: false as const, message: 'dbt project not found' })); + const testPrompts = prompts({ + multiselect: [['dbt']], + select: ['edit:dbt-main', 'path'], + text: ['/repo/new-dbt', ''], + }); + + await expect( + runKtxSetupSourcesStep( + { projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false }, + makeIo().io, + { + prompts: testPrompts, + validateDbt, + }, + ), + ).resolves.toEqual({ status: 'failed', projectDir }); + + expect(validateDbt).toHaveBeenCalledWith(expect.objectContaining({ + driver: 'dbt', + source_dir: '/repo/new-dbt', + })); + const config = await readConfig(); + expect(config.connections['dbt-main']).toMatchObject({ + driver: 'dbt', + source_dir: '/repo/existing-dbt', + project_name: 'analytics', + }); + expect(config.ingest.adapters).not.toContain('dbt'); + }); + + it('lets git-backed context source edits keep the existing repo credential', async () => { + await addPrimarySource(); + await addConnection('metricflow-main', { + driver: 'metricflow', + metricflow: { + repoUrl: 'https://github.com/acme/private-metricflow', + branch: 'main', + path: 'metrics', + auth_token_ref: 'env:METRICFLOW_REPO_TOKEN', // pragma: allowlist secret + }, + }); + const testGitRepo = vi.fn(async () => ({ ok: false as const, error: 'authentication required' })); + const testPrompts = prompts({ + multiselect: [['metricflow']], + select: ['edit:metricflow-main', 'git', 'keep', 'done'], + text: ['https://github.com/acme/private-metricflow', 'main', 'metrics'], + }); + + await expect( + runKtxSetupSourcesStep( + { projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false }, + makeIo().io, + { + prompts: testPrompts, + testGitRepo, + validateMetricflow: vi.fn(async () => ({ ok: true as const, detail: 'metrics=1' })), + }, + ), + ).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['metricflow-main'] }); + + expect(testPrompts.select).toHaveBeenCalledWith({ + message: 'This MetricFlow repo requires authentication.', + options: [ + { value: 'keep', label: 'Keep existing credential' }, + { value: 'env', label: 'Use GITHUB_TOKEN from the environment' }, + { value: 'paste', label: 'Paste a token and save it as a local secret file' }, + { value: 'skip', label: 'Skip — try without authentication' }, + { value: 'back', label: 'Back' }, + ], + }); + expect((await readConfig()).connections['metricflow-main']).toMatchObject({ + driver: 'metricflow', + metricflow: { + repoUrl: 'https://github.com/acme/private-metricflow', + branch: 'main', + path: 'metrics', + auth_token_ref: 'env:METRICFLOW_REPO_TOKEN', + }, + }); + }); + + it('edits an existing context source from the configured-source follow-up menu', async () => { + await addPrimarySource(); + await addConnection('dbt-main', { + driver: 'dbt', + source_dir: '/repo/existing-dbt', + project_name: 'analytics', + }); + const validateDbt = vi.fn(async () => ({ ok: true as const, detail: 'project=analytics schemas=2' })); + const testPrompts = prompts({ + multiselect: [['dbt']], + select: ['existing:dbt-main', 'edit', 'dbt-main', 'path', 'done'], + text: ['/repo/edited-dbt', ''], + }); + + await expect( + runKtxSetupSourcesStep( + { projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false }, + makeIo().io, + { + prompts: testPrompts, + validateDbt, + }, + ), + ).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['dbt-main'] }); + + expect(testPrompts.select).toHaveBeenCalledWith({ + message: '1 context source configured (dbt-main). Add another?', + options: [ + { value: 'done', label: 'Done — continue to context build' }, + { value: 'edit', label: 'Edit an existing context source' }, + { value: 'add', label: 'Add another context source' }, + ], + }); + expect(testPrompts.select).toHaveBeenCalledWith({ + message: 'Context source to edit', + options: [ + { value: 'dbt-main', label: 'dbt-main (dbt)' }, + { value: 'back', label: 'Back' }, + ], + }); + expect(testPrompts.text).toHaveBeenCalledWith({ + message: textInputPrompt('dbt local path'), + initialValue: '/repo/existing-dbt', + }); + expect(validateDbt).toHaveBeenLastCalledWith(expect.objectContaining({ + driver: 'dbt', + source_dir: '/repo/edited-dbt', + })); + expect((await readConfig()).connections['dbt-main']).toMatchObject({ + driver: 'dbt', + source_dir: '/repo/edited-dbt', + project_name: 'analytics', + }); + }); + + it('backs out of editing an existing context source to the source connection menu', async () => { + await addPrimarySource(); + await addConnection('dbt-main', { + driver: 'dbt', + source_dir: '/repo/existing-dbt', + project_name: 'analytics', + }); + const validateDbt = vi.fn(async () => ({ ok: true as const, detail: 'project=analytics schemas=2' })); + const testPrompts = prompts({ + multiselect: [['dbt']], + select: ['edit:dbt-main', 'back', 'existing:dbt-main'], + }); + + await expect( + runKtxSetupSourcesStep( + { projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false }, + makeIo().io, + { + prompts: testPrompts, + validateDbt, + }, + ), + ).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['dbt-main'] }); + + expect( + vi + .mocked(testPrompts.select) + .mock.calls.map(([options]) => options.message) + .filter((message) => message === 'Configure dbt'), + ).toHaveLength(2); + expect(validateDbt).toHaveBeenCalledWith({ + driver: 'dbt', + source_dir: '/repo/existing-dbt', + project_name: 'analytics', + }); + expect((await readConfig()).connections['dbt-main']).toMatchObject({ + driver: 'dbt', + source_dir: '/repo/existing-dbt', + project_name: 'analytics', + }); + }); + it('lets Escape from dbt git URL return to source location selection', async () => { await addPrimarySource(); const validateDbt = vi.fn(async () => ({ ok: true as const, detail: 'project=analytics schemas=2' })); diff --git a/packages/cli/src/setup-sources.ts b/packages/cli/src/setup-sources.ts index 65a8b09b..359cf0d8 100644 --- a/packages/cli/src/setup-sources.ts +++ b/packages/cli/src/setup-sources.ts @@ -156,6 +156,10 @@ function sourceLabel(source: KtxSetupSourceType): string { return SOURCE_LABELS[source]; } +function sourceAdapter(source: KtxSetupSourceType): string { + return source; +} + function connectionNamePrompt(label: string): string { return `Name this ${label} connection\nKTX will use this short name in commands and config. You can rename it now.`; } @@ -220,17 +224,20 @@ async function chooseSourceCredentialRef(input: { label: string; envName: string; secretFileName: string; + existingRef?: string; }): Promise { while (true) { const choice = await input.prompts.select({ message: `How should KTX find your ${input.label}?`, options: [ + ...(input.existingRef ? [{ value: 'keep', label: 'Keep existing credential' }] : []), { value: 'env', label: `Use ${input.envName} from the environment` }, { value: 'paste', label: 'Paste a key and save it as a local secret file' }, { value: 'back', label: 'Back' }, ], }); if (choice === 'back') return 'back'; + if (choice === 'keep' && input.existingRef) return input.existingRef; if (choice === 'paste') { const value = await input.prompts.password({ message: input.label }); if (value === undefined) continue; @@ -252,12 +259,14 @@ async function chooseGitAuthCredentialRef(input: { projectDir: string; source: KtxSetupSourceType; connectionId: string; + existingRef?: string; }): Promise { const label = input.source === 'dbt' ? 'This' : `This ${sourceLabel(input.source)}`; while (true) { const choice = await input.prompts.select({ message: `${label} repo requires authentication.`, options: [ + ...(input.existingRef ? [{ value: 'keep', label: 'Keep existing credential' }] : []), { value: 'env', label: 'Use GITHUB_TOKEN from the environment' }, { value: 'paste', label: 'Paste a token and save it as a local secret file' }, { value: 'skip', label: 'Skip — try without authentication' }, @@ -265,6 +274,7 @@ async function chooseGitAuthCredentialRef(input: { ], }); if (choice === 'back') return 'back'; + if (choice === 'keep' && input.existingRef) return input.existingRef; if (choice === 'skip') return undefined; if (choice === 'paste') { const value = await input.prompts.password({ message: 'Git access token' }); @@ -309,17 +319,25 @@ async function writeSourceConnection( projectDir: string, connectionId: string, connection: KtxProjectConnectionConfig, + adapter: string, ): Promise<() => Promise> { assertSafeConnectionId(connectionId); const project = await loadKtxProject({ projectDir }); const previousConnection = project.config.connections[connectionId]; const hadPreviousConnection = previousConnection !== undefined; + const shouldRemoveAdapterOnRollback = !project.config.ingest.adapters.includes(adapter); const config = { ...project.config, connections: { ...project.config.connections, [connectionId]: connection, }, + ingest: { + ...project.config.ingest, + adapters: project.config.ingest.adapters.includes(adapter) + ? [...project.config.ingest.adapters] + : [...project.config.ingest.adapters, adapter], + }, }; await writeFile(project.configPath, serializeKtxProjectConfig(config), 'utf-8'); return async () => { @@ -333,10 +351,31 @@ async function writeSourceConnection( await writeProjectConfig(projectDir, { ...latest.config, connections, + ingest: { + ...latest.config.ingest, + adapters: shouldRemoveAdapterOnRollback + ? latest.config.ingest.adapters.filter((candidate) => candidate !== adapter) + : latest.config.ingest.adapters, + }, }); }; } +async function ensureSourceAdapterEnabled(projectDir: string, source: KtxSetupSourceType): Promise { + const adapter = sourceAdapter(source); + const project = await loadKtxProject({ projectDir }); + if (project.config.ingest.adapters.includes(adapter)) { + return; + } + await writeProjectConfig(projectDir, { + ...project.config, + ingest: { + ...project.config.ingest, + adapters: [...project.config.ingest.adapters, adapter], + }, + }); +} + async function markSourcesComplete(projectDir: string): Promise { const project = await loadKtxProject({ projectDir }); await writeFile(project.configPath, serializeKtxProjectConfig(project.config), 'utf-8'); @@ -760,8 +799,14 @@ interface WarehouseConnectionChoice { type InteractiveSourceConnectionChoice = | { kind: 'existing'; connectionId: string; connection: KtxProjectConnectionConfig } | { kind: 'new'; args: KtxSetupSourcesArgs } + | { kind: 'edited'; connectionId: string; args: KtxSetupSourcesArgs } | 'back'; +type SourceSetupChoiceResult = + | { status: 'ready'; connectionId: string } + | { status: 'back' } + | { status: 'failed' }; + async function runSourcePromptSteps( initialState: SourcePromptState, stepsForState: (state: SourcePromptState) => SourcePromptStep[], @@ -795,6 +840,12 @@ function resetRepoLocationFields(state: SourcePromptState): void { delete state.sourceProjectName; } +function sourceLocationFromArgs(args: KtxSetupSourcesArgs): SourceLocationChoice | undefined { + if (args.sourcePath) return 'path'; + if (args.sourceGitUrl) return 'git'; + return undefined; +} + function warehouseConnectionChoices(config: KtxProjectConfig): WarehouseConnectionChoice[] { return Object.entries(config.connections) .filter(([, connection]) => PRIMARY_SOURCE_DRIVERS.has(String(connection.driver ?? '').toLowerCase())) @@ -931,7 +982,7 @@ async function promptForInteractiveSource( testGitRepo: KtxSetupSourcesDeps['testGitRepo'] = testRepoConnection, discoverMetabaseDatabaseList?: KtxSetupSourcesDeps['discoverMetabaseDatabases'], ): Promise { - const initialState: SourcePromptState = { ...args, source }; + const initialState: SourcePromptState = { ...args, source, sourceLocation: sourceLocationFromArgs(args) }; if (args.sourceConnectionId) { initialState.sourceConnectionId = args.sourceConnectionId; } @@ -961,7 +1012,10 @@ async function promptForInteractiveSource( ...(state.sourceLocation === 'path' ? [ async (currentState: SourcePromptState) => { - const sourcePath = await promptText(prompts, { message: `${source} local path` }); + const sourcePath = await promptText(prompts, { + message: `${source} local path`, + ...(currentState.sourcePath ? { initialValue: currentState.sourcePath } : {}), + }); if (sourcePath === undefined) return 'back'; currentState.sourcePath = sourcePath; return 'next'; @@ -971,13 +1025,19 @@ async function promptForInteractiveSource( ...(state.sourceLocation === 'git' ? [ async (currentState: SourcePromptState) => { - const sourceGitUrl = await promptText(prompts, { message: `${source} git URL` }); + const sourceGitUrl = await promptText(prompts, { + message: `${source} git URL`, + ...(currentState.sourceGitUrl ? { initialValue: currentState.sourceGitUrl } : {}), + }); if (sourceGitUrl === undefined) return 'back'; currentState.sourceGitUrl = sourceGitUrl; return 'next'; }, async (currentState: SourcePromptState) => { - const branch = await promptText(prompts, { message: `${source} git branch`, initialValue: 'main' }); + const branch = await promptText(prompts, { + message: `${source} git branch`, + initialValue: currentState.sourceBranch ?? 'main', + }); if (branch === undefined) return 'back'; currentState.sourceBranch = branch || 'main'; return 'next'; @@ -998,6 +1058,7 @@ async function promptForInteractiveSource( projectDir: args.projectDir, source, connectionId: currentState.sourceConnectionId ?? `${source}-main`, + existingRef: currentState.sourceAuthTokenRef, }); if (authRef === 'back') return 'back'; if (authRef) { @@ -1071,6 +1132,7 @@ async function promptForInteractiveSource( const subpath = await promptText(prompts, { message: sourceSubpathPrompt(source), placeholder: 'optional', + ...(currentState.sourceSubpath ? { initialValue: currentState.sourceSubpath } : {}), }); if (subpath === undefined) return 'back'; if (subpath) { @@ -1089,7 +1151,10 @@ async function promptForInteractiveSource( return await runSourcePromptSteps(initialState, () => [ ...connectionSteps, async (state) => { - const sourceUrl = await promptText(prompts, { message: 'Metabase URL' }); + const sourceUrl = await promptText(prompts, { + message: 'Metabase URL', + ...(state.sourceUrl ? { initialValue: state.sourceUrl } : {}), + }); if (sourceUrl === undefined) return 'back'; state.sourceUrl = sourceUrl; return 'next'; @@ -1101,6 +1166,7 @@ async function promptForInteractiveSource( label: 'Metabase API key', envName: 'METABASE_API_KEY', secretFileName: `${state.sourceConnectionId ?? 'metabase-main'}-api-key`, + existingRef: state.sourceApiKeyRef, }); if (ref === 'back') return 'back'; state.sourceApiKeyRef = ref; @@ -1132,13 +1198,19 @@ async function promptForInteractiveSource( return await runSourcePromptSteps(initialState, () => [ ...connectionSteps, async (state) => { - const sourceUrl = await promptText(prompts, { message: 'Looker base URL' }); + const sourceUrl = await promptText(prompts, { + message: 'Looker base URL', + ...(state.sourceUrl ? { initialValue: state.sourceUrl } : {}), + }); if (sourceUrl === undefined) return 'back'; state.sourceUrl = sourceUrl; return 'next'; }, async (state) => { - const sourceClientId = await promptText(prompts, { message: 'Looker client id' }); + const sourceClientId = await promptText(prompts, { + message: 'Looker client id', + ...(state.sourceClientId ? { initialValue: state.sourceClientId } : {}), + }); if (sourceClientId === undefined) return 'back'; state.sourceClientId = sourceClientId; return 'next'; @@ -1150,6 +1222,7 @@ async function promptForInteractiveSource( label: 'Looker client secret', envName: 'LOOKER_CLIENT_SECRET', secretFileName: `${state.sourceConnectionId ?? 'looker-main'}-client-secret`, + existingRef: state.sourceClientSecretRef, }); if (ref === 'back') return 'back'; state.sourceClientSecretRef = ref; @@ -1168,6 +1241,7 @@ async function promptForInteractiveSource( const lookerConnectionName = await promptText(prompts, { message: 'Looker connection name', placeholder: 'optional', + ...(state.sourceTarget ? { initialValue: state.sourceTarget } : {}), }); if (lookerConnectionName === undefined) return 'back'; if (lookerConnectionName) { @@ -1189,6 +1263,7 @@ async function promptForInteractiveSource( label: 'Notion integration token', envName: 'NOTION_TOKEN', secretFileName: `${currentState.sourceConnectionId ?? 'notion-main'}-token`, + existingRef: currentState.sourceApiKeyRef, }); if (ref === 'back') return 'back'; currentState.sourceApiKeyRef = ref; @@ -1253,6 +1328,24 @@ function existingConnectionIdsBySource( .sort((left, right) => left.localeCompare(right)); } +function sourceTypeForConnection(connection: KtxProjectConnectionConfig): KtxSetupSourceType | null { + const driver = String(connection.driver ?? '').toLowerCase(); + return SOURCE_OPTIONS.some((option) => option.value === driver) ? (driver as KtxSetupSourceType) : null; +} + +function contextSourceEditTargets(connections: Record): Array<{ + connectionId: string; + source: KtxSetupSourceType; +}> { + return Object.entries(connections) + .map(([connectionId, connection]) => { + const source = sourceTypeForConnection(connection); + return source ? { connectionId, source } : null; + }) + .filter((target): target is { connectionId: string; source: KtxSetupSourceType } => target !== null) + .sort((left, right) => left.connectionId.localeCompare(right.connectionId)); +} + function sourceChecklistForConnections(connections: Record): { options: Array<{ value: KtxSetupSourceType; label: string; hint?: string }>; initialValues: KtxSetupSourceType[]; @@ -1284,6 +1377,180 @@ function defaultConnectionIdForSource( return `${base}-${index}`; } +function firstStringRecordEntry(value: unknown): [string, string] | undefined { + if (!isRecord(value)) return undefined; + for (const [key, raw] of Object.entries(value)) { + if (typeof raw === 'string' && raw.trim().length > 0) { + return [key, raw.trim()]; + } + } + return undefined; +} + +function applyRepoSourceArgs( + args: KtxSetupSourcesArgs, + input: { repoUrl?: string; sourceDir?: string; branch?: string; subpath?: string; authTokenRef?: string }, +): void { + if (input.sourceDir) { + args.sourcePath = input.sourceDir; + } else if (input.repoUrl?.startsWith('file:')) { + args.sourcePath = fileURLToPath(input.repoUrl); + } else if (input.repoUrl) { + args.sourceGitUrl = input.repoUrl; + } + if (input.branch) args.sourceBranch = input.branch; + if (input.subpath) args.sourceSubpath = input.subpath; + if (input.authTokenRef) args.sourceAuthTokenRef = input.authTokenRef; +} + +function sourceArgsFromExistingConnection(input: { + args: KtxSetupSourcesArgs; + source: KtxSetupSourceType; + connectionId: string; + connection: KtxProjectConnectionConfig; +}): KtxSetupSourcesArgs { + const sourceArgs: KtxSetupSourcesArgs = { + projectDir: input.args.projectDir, + inputMode: input.args.inputMode, + source: input.source, + sourceConnectionId: input.connectionId, + runInitialSourceIngest: input.args.runInitialSourceIngest, + skipSources: input.args.skipSources, + }; + + if (input.source === 'dbt') { + applyRepoSourceArgs(sourceArgs, { + sourceDir: stringField(input.connection.source_dir), + repoUrl: stringField(input.connection.repo_url), + branch: stringField(input.connection.branch), + subpath: stringField(input.connection.path), + authTokenRef: stringField(input.connection.auth_token_ref), + }); + const profilesPath = stringField(input.connection.profiles_path); + const target = stringField(input.connection.target); + const projectName = stringField(input.connection.project_name); + if (profilesPath) sourceArgs.sourceProfilesPath = profilesPath; + if (target) sourceArgs.sourceTarget = target; + if (projectName) sourceArgs.sourceProjectName = projectName; + return sourceArgs; + } + + if (input.source === 'metricflow') { + const metricflow = isRecord(input.connection.metricflow) ? input.connection.metricflow : {}; + applyRepoSourceArgs(sourceArgs, { + repoUrl: stringField(metricflow.repoUrl), + branch: stringField(metricflow.branch), + subpath: stringField(metricflow.path), + authTokenRef: stringField(metricflow.auth_token_ref), + }); + return sourceArgs; + } + + if (input.source === 'lookml') { + applyRepoSourceArgs(sourceArgs, { + repoUrl: stringField(input.connection.repoUrl), + branch: stringField(input.connection.branch), + subpath: stringField(input.connection.path), + authTokenRef: stringField(input.connection.auth_token_ref), + }); + const mappings = isRecord(input.connection.mappings) ? input.connection.mappings : {}; + const expectedLookerConnectionName = stringField(mappings.expectedLookerConnectionName); + if (expectedLookerConnectionName) sourceArgs.sourceTarget = expectedLookerConnectionName; + return sourceArgs; + } + + if (input.source === 'metabase') { + sourceArgs.sourceUrl = stringField(input.connection.api_url); + sourceArgs.sourceApiKeyRef = stringField(input.connection.api_key_ref); + const mappings = isRecord(input.connection.mappings) ? input.connection.mappings : {}; + const databaseMapping = firstStringRecordEntry(mappings.databaseMappings); + if (databaseMapping) { + sourceArgs.metabaseDatabaseId = Number.parseInt(databaseMapping[0], 10); + sourceArgs.sourceWarehouseConnectionId = databaseMapping[1]; + } + return sourceArgs; + } + + if (input.source === 'looker') { + sourceArgs.sourceUrl = stringField(input.connection.base_url); + sourceArgs.sourceClientId = stringField(input.connection.client_id); + sourceArgs.sourceClientSecretRef = stringField(input.connection.client_secret_ref); + const mappings = isRecord(input.connection.mappings) ? input.connection.mappings : {}; + const connectionMapping = firstStringRecordEntry(mappings.connectionMappings); + if (connectionMapping) { + sourceArgs.sourceTarget = connectionMapping[0]; + sourceArgs.sourceWarehouseConnectionId = connectionMapping[1]; + } + return sourceArgs; + } + + sourceArgs.sourceApiKeyRef = stringField(input.connection.auth_token_ref); + sourceArgs.notionCrawlMode = + input.connection.crawl_mode === 'all_accessible' ? 'all_accessible' : 'selected_roots'; + if (Array.isArray(input.connection.root_page_ids)) { + sourceArgs.notionRootPageIds = input.connection.root_page_ids.filter( + (pageId): pageId is string => typeof pageId === 'string', + ); + } + return sourceArgs; +} + +async function promptEditedSourceConnection(input: { + args: KtxSetupSourcesArgs; + source: KtxSetupSourceType; + connectionId: string; + connection: KtxProjectConnectionConfig; + prompts: KtxSetupSourcesPromptAdapter; + io: KtxCliIo; + testGitRepo?: KtxSetupSourcesDeps['testGitRepo']; + pickNotionRootPages?: KtxSetupSourcesDeps['pickNotionRootPages']; + discoverMetabaseDatabases?: KtxSetupSourcesDeps['discoverMetabaseDatabases']; +}): Promise | 'back'> { + const sourceArgs = await promptForInteractiveSource( + sourceArgsFromExistingConnection({ + args: input.args, + source: input.source, + connectionId: input.connectionId, + connection: input.connection, + }), + input.source, + input.prompts, + input.io, + { + pickNotionRootPages: input.pickNotionRootPages, + discoverMetabaseDatabases: input.discoverMetabaseDatabases, + }, + input.connectionId, + input.testGitRepo, + input.discoverMetabaseDatabases, + ); + return sourceArgs === 'back' + ? 'back' + : { kind: 'edited', connectionId: input.connectionId, args: sourceArgs }; +} + +async function chooseContextSourceToEdit(input: { + projectDir: string; + prompts: KtxSetupSourcesPromptAdapter; +}): Promise<{ connectionId: string; source: KtxSetupSourceType } | 'back'> { + const project = await loadKtxProject({ projectDir: input.projectDir }); + const targets = contextSourceEditTargets(project.config.connections); + if (targets.length === 0) return 'back'; + const choice = await input.prompts.select({ + message: 'Context source to edit', + options: [ + ...targets.map((target) => ({ + value: target.connectionId, + label: `${target.connectionId} (${sourceLabel(target.source)})`, + })), + { value: 'back', label: 'Back' }, + ], + }); + if (choice === 'back') return 'back'; + const target = targets.find((candidate) => candidate.connectionId === choice); + return target ?? 'back'; +} + async function chooseInteractiveSourceConnection(input: { args: KtxSetupSourcesArgs; source: KtxSetupSourceType; @@ -1323,6 +1590,10 @@ async function chooseInteractiveSourceConnection(input: { value: `existing:${connectionId}`, label: `Use existing ${label} connection: ${connectionId}`, })), + ...existingIds.map((connectionId) => ({ + value: `edit:${connectionId}`, + label: `Edit existing ${label} connection: ${connectionId}`, + })), { value: 'new', label: `Add new ${label} connection` }, { value: 'back', label: 'Back' }, ], @@ -1336,6 +1607,28 @@ async function chooseInteractiveSourceConnection(input: { } continue; } + if (choice.startsWith('edit:')) { + const connectionId = choice.slice('edit:'.length); + const connection = input.connections[connectionId]; + if (!connection) { + continue; + } + const edited = await promptEditedSourceConnection({ + args: input.args, + source: input.source, + connectionId, + connection, + prompts: input.prompts, + io: input.io, + testGitRepo: input.testGitRepo, + pickNotionRootPages: input.pickNotionRootPages, + discoverMetabaseDatabases: input.discoverMetabaseDatabases, + }); + if (edited === 'back') { + continue; + } + return edited; + } const sourceArgs = await promptForInteractiveSource( input.args, input.source, @@ -1400,6 +1693,85 @@ async function validateSource( return await (deps.validateNotion ?? defaultValidateNotion)(args.connection); } +async function saveValidateAndMaybeBuildSource(input: { + args: KtxSetupSourcesArgs; + source: KtxSetupSourceType; + sourceChoice: Exclude; + prompts: KtxSetupSourcesPromptAdapter; + io: KtxCliIo; + deps: KtxSetupSourcesDeps; +}): Promise { + const connectionId = + input.sourceChoice.kind === 'existing' + ? input.sourceChoice.connectionId + : input.sourceChoice.kind === 'edited' + ? input.sourceChoice.connectionId + : (input.sourceChoice.args.sourceConnectionId ?? `${input.source}-main`); + const connection = + input.sourceChoice.kind === 'existing' + ? input.sourceChoice.connection + : buildConnection(input.source, input.sourceChoice.args); + const rollback = + input.sourceChoice.kind === 'existing' + ? undefined + : await writeSourceConnection( + input.args.projectDir, + connectionId, + connection, + sourceAdapter(input.source), + ); + + if (input.sourceChoice.kind === 'existing') { + await ensureSourceAdapterEnabled(input.args.projectDir, input.source); + } + + const validation = await validateSource( + input.source, + { projectDir: input.args.projectDir, connectionId, connection }, + input.deps, + ); + if (!validation.ok) { + await rollback?.(); + input.io.stderr.write(`${validation.message}\n`); + return { status: 'failed' }; + } + + if (input.source === 'metabase' || input.source === 'looker') { + input.prompts.log?.(`Validating ${sourceLabel(input.source)} mapping…`); + const mappingCode = await (input.deps.runMapping ?? defaultRunMapping)( + input.args.projectDir, + connectionId, + createSetupPrefixedIo(input.io), + ); + if (mappingCode !== 0) { + await rollback?.(); + return { status: 'failed' }; + } + } + + if (input.args.runInitialSourceIngest) { + const ingestResult = await runInitialSourceIngestWithRecovery({ + args: input.args, + connectionId, + io: input.io, + prompts: input.prompts, + deps: input.deps, + }); + if (ingestResult === 'failed') { + await rollback?.(); + return { status: 'failed' }; + } + if (ingestResult === 'back') { + await rollback?.(); + return { status: 'back' }; + } + } else { + input.io.stdout.write(`│ Context source ${connectionId} saved. It will be built during the context build step.\n`); + } + + return { status: 'ready', connectionId }; +} + export async function runKtxSetupSourcesStep( args: KtxSetupSourcesArgs, io: KtxCliIo, @@ -1477,59 +1849,27 @@ export async function runKtxSetupSourcesStep( returnToSourceSelection = true; break; } - const connectionId = - sourceChoice.kind === 'existing' - ? sourceChoice.connectionId - : (sourceChoice.args.sourceConnectionId ?? `${source}-main`); - const connection = - sourceChoice.kind === 'existing' ? sourceChoice.connection : buildConnection(source, sourceChoice.args); - const rollback = - sourceChoice.kind === 'existing' - ? undefined - : await writeSourceConnection(args.projectDir, connectionId, connection); - const validation = await validateSource(source, { projectDir: args.projectDir, connectionId, connection }, deps); - - if (!validation.ok) { - await rollback?.(); - io.stderr.write(`${validation.message}\n`); + const choiceResult = await saveValidateAndMaybeBuildSource({ + args, + source, + sourceChoice, + prompts, + io, + deps, + }); + if (choiceResult.status === 'failed') { return { status: 'failed', projectDir: args.projectDir }; } - if (source === 'metabase' || source === 'looker') { - prompts.log?.(`Validating ${sourceLabel(source)} mapping…`); - const mappingCode = await (deps.runMapping ?? defaultRunMapping)( - args.projectDir, - connectionId, - createSetupPrefixedIo(io), - ); - if (mappingCode !== 0) { - await rollback?.(); - return { status: 'failed', projectDir: args.projectDir }; + if (choiceResult.status === 'back') { + if (args.source) { + return { status: 'back', projectDir: args.projectDir }; } + returnToSourceSelection = true; + break; } - if (args.runInitialSourceIngest) { - const ingestResult = await runInitialSourceIngestWithRecovery({ - args, - connectionId, - io, - prompts, - deps, - }); - if (ingestResult === 'failed') { - await rollback?.(); - return { status: 'failed', projectDir: args.projectDir }; - } - if (ingestResult === 'back') { - await rollback?.(); - if (args.source) { - return { status: 'back', projectDir: args.projectDir }; - } - returnToSourceSelection = true; - break; - } - } else { - io.stdout.write(`│ Context source ${connectionId} saved. It will be built during the context build step.\n`); + if (!readyConnectionIds.includes(choiceResult.connectionId)) { + readyConnectionIds.push(choiceResult.connectionId); } - readyConnectionIds.push(connectionId); } if (returnToSourceSelection) { @@ -1537,14 +1877,66 @@ export async function runKtxSetupSourcesStep( } if (readyConnectionIds.length > 0 && !args.source && args.inputMode !== 'disabled') { - const addMore = await prompts.select({ - message: `${readyConnectionIds.length} context source${readyConnectionIds.length > 1 ? 's' : ''} configured (${readyConnectionIds.join(', ')}). Add another?`, - options: [ - { value: 'done', label: 'Done — continue to context build' }, - { value: 'add', label: 'Add another context source' }, - ], - }); - if (addMore === 'add') { + let restartSourceSelection = false; + while (true) { + const addMore = await prompts.select({ + message: `${readyConnectionIds.length} context source${readyConnectionIds.length > 1 ? 's' : ''} configured (${readyConnectionIds.join(', ')}). Add another?`, + options: [ + { value: 'done', label: 'Done — continue to context build' }, + { value: 'edit', label: 'Edit an existing context source' }, + { value: 'add', label: 'Add another context source' }, + ], + }); + if (addMore === 'add') { + restartSourceSelection = true; + break; + } + if (addMore === 'edit') { + const editTarget = await chooseContextSourceToEdit({ projectDir: args.projectDir, prompts }); + if (editTarget === 'back') { + continue; + } + const projectForEdit = await loadKtxProject({ projectDir: args.projectDir }); + const connection = projectForEdit.config.connections[editTarget.connectionId]; + if (!connection) { + continue; + } + const sourceChoice = await promptEditedSourceConnection({ + args, + source: editTarget.source, + connectionId: editTarget.connectionId, + connection, + prompts, + io, + testGitRepo: deps.testGitRepo, + pickNotionRootPages: deps.pickNotionRootPages, + discoverMetabaseDatabases: deps.discoverMetabaseDatabases, + }); + if (sourceChoice === 'back') { + continue; + } + const choiceResult = await saveValidateAndMaybeBuildSource({ + args, + source: editTarget.source, + sourceChoice, + prompts, + io, + deps, + }); + if (choiceResult.status === 'failed') { + return { status: 'failed', projectDir: args.projectDir }; + } + if (choiceResult.status === 'back') { + continue; + } + if (!readyConnectionIds.includes(choiceResult.connectionId)) { + readyConnectionIds.push(choiceResult.connectionId); + } + continue; + } + break; + } + if (restartSourceSelection) { continue; } } diff --git a/packages/cli/src/setup.test.ts b/packages/cli/src/setup.test.ts index b1cec14a..b0c28d7c 100644 --- a/packages/cli/src/setup.test.ts +++ b/packages/cli/src/setup.test.ts @@ -170,7 +170,6 @@ describe('setup status', () => { ' warehouse:', ' driver: postgres', ' url: env:DATABASE_URL', - ' readonly: true', '', ].join('\n'), 'utf-8', @@ -192,7 +191,6 @@ describe('setup status', () => { ' warehouse:', ' driver: postgres', ' url: env:DATABASE_URL', - ' readonly: true', '', ].join('\n'), 'utf-8', @@ -1385,7 +1383,6 @@ describe('setup status', () => { ' warehouse:', ' driver: postgres', ' url: env:DEMO_DATABASE_URL', - ' readonly: true', '', ].join('\n'), 'utf-8', diff --git a/packages/cli/src/sl.test.ts b/packages/cli/src/sl.test.ts index ff4132b4..14f18337 100644 --- a/packages/cli/src/sl.test.ts +++ b/packages/cli/src/sl.test.ts @@ -190,7 +190,7 @@ joins: [] it('runs sl query and prints SQL output', async () => { const projectDir = join(tempDir, 'project'); const project = await initKtxProject({ projectDir, projectName: 'warehouse' }); - project.config.connections.warehouse = { driver: 'postgres', readonly: true }; + project.config.connections.warehouse = { driver: 'postgres' }; await project.fileStore.writeFile( 'semantic-layer/warehouse/orders.yaml', `name: orders @@ -247,7 +247,7 @@ joins: [] it('runs sl query from a JSON query file', async () => { const projectDir = join(tempDir, 'project'); const project = await initKtxProject({ projectDir, projectName: 'warehouse' }); - project.config.connections.warehouse = { driver: 'postgres', readonly: true }; + project.config.connections.warehouse = { driver: 'postgres' }; await project.fileStore.writeFile( 'semantic-layer/warehouse/orders.yaml', `name: orders @@ -314,7 +314,7 @@ joins: [] it('creates default sl query compute through the managed runtime helper', async () => { const projectDir = join(tempDir, 'project'); const project = await initKtxProject({ projectDir, projectName: 'warehouse' }); - project.config.connections.warehouse = { driver: 'postgres', readonly: true }; + project.config.connections.warehouse = { driver: 'postgres' }; await project.fileStore.writeFile( 'semantic-layer/warehouse/orders.yaml', `name: orders @@ -375,7 +375,7 @@ joins: [] it('executes sl query through the injected query executor', async () => { const projectDir = join(tempDir, 'project'); const project = await initKtxProject({ projectDir, projectName: 'warehouse' }); - project.config.connections.warehouse = { driver: 'postgres', url: 'postgres://example/db', readonly: true }; + project.config.connections.warehouse = { driver: 'postgres', url: 'postgres://example/db' }; await project.fileStore.writeFile( 'semantic-layer/warehouse/orders.yaml', `name: orders @@ -471,7 +471,7 @@ joins: [] `); db.close(); - project.config.connections.warehouse = { driver: 'sqlite', path: 'warehouse.db', readonly: true }; + project.config.connections.warehouse = { driver: 'sqlite', path: 'warehouse.db' }; await writeFile( join(projectDir, 'ktx.yaml'), [ @@ -480,7 +480,6 @@ joins: [] ' warehouse:', ' driver: sqlite', ' path: warehouse.db', - ' readonly: true', '', ].join('\n'), 'utf-8', diff --git a/packages/cli/src/standalone-smoke.test.ts b/packages/cli/src/standalone-smoke.test.ts index 19bf02c5..bfd6bdba 100644 --- a/packages/cli/src/standalone-smoke.test.ts +++ b/packages/cli/src/standalone-smoke.test.ts @@ -84,7 +84,6 @@ async function writeSqliteScanConfig(projectDir: string, dbPath: string, enrich ' warehouse:', ' driver: sqlite', ` path: ${JSON.stringify(dbPath)}`, - ' readonly: true', 'ingest:', ' adapters:', ' - live-database', @@ -108,6 +107,10 @@ function expectProjectStderr(result: CliResult, projectDir: string): void { expect(result).toMatchObject({ code: 0, stderr: `Project: ${projectDir}\n` }); } +function expectSetupStderr(result: CliResult): void { + expect(result).toMatchObject({ code: 0, stderr: '' }); +} + async function runSetupNewProject(projectDir: string): Promise { return await runBuiltCli([ 'setup', @@ -139,7 +142,7 @@ describe('standalone built ktx CLI smoke', () => { const projectDir = join(tempDir, 'project'); const init = await runSetupNewProject(projectDir); - expectProjectStderr(init, projectDir); + expectSetupStderr(init); expect(init.stdout).toContain(`Project: ${projectDir}`); const run = await runBuiltCli([ @@ -175,7 +178,7 @@ describe('standalone built ktx CLI smoke', () => { it('runs fast public database ingest through the built binary with manifest artifacts', async () => { const projectDir = join(tempDir, 'database-ingest-project'); const init = await runSetupNewProject(projectDir); - expectProjectStderr(init, projectDir); + expectSetupStderr(init); const dbPath = join(projectDir, 'warehouse.db'); createSqliteWarehouse(dbPath); @@ -254,7 +257,7 @@ describe('standalone built ktx CLI smoke', () => { it('rejects the removed connection add command through the built binary', async () => { const projectDir = join(tempDir, 'notion-project'); const init = await runSetupNewProject(projectDir); - expectProjectStderr(init, projectDir); + expectSetupStderr(init); const add = await runBuiltCli([ 'connection', diff --git a/packages/cli/src/text-ingest.test.ts b/packages/cli/src/text-ingest.test.ts new file mode 100644 index 00000000..55dbe9e3 --- /dev/null +++ b/packages/cli/src/text-ingest.test.ts @@ -0,0 +1,339 @@ +import { describe, expect, it, vi } from 'vitest'; +import type { MemoryCaptureStatus } from '@ktx/context/memory'; +import type { KtxLocalProject } from '@ktx/context/project'; +import { runKtxTextIngest, type TextMemoryCapturePort } from './text-ingest.js'; + +function makeIo(options: { isTTY?: boolean } = {}) { + let stdout = ''; + let stderr = ''; + return { + io: { + stdout: { + isTTY: options.isTTY, + write: (chunk: string) => { + stdout += chunk; + }, + }, + stderr: { + write: (chunk: string) => { + stderr += chunk; + }, + }, + }, + stdout: () => stdout, + stderr: () => stderr, + }; +} + +function fakeCapture( + options: { + failRunIds?: Set; + missingStatusRunIds?: Set; + events?: string[]; + } = {}, +): TextMemoryCapturePort { + let next = 1; + return { + capture: vi.fn(async () => { + const runId = `run-${next++}`; + options.events?.push(`capture:${runId}`); + return { runId }; + }), + waitForRun: vi.fn(async (runId: string) => { + options.events?.push(`wait:${runId}`); + }), + status: vi.fn(async (runId: string) => { + options.events?.push(`status:${runId}`); + if (options.missingStatusRunIds?.has(runId)) { + return null; + } + if (options.failRunIds?.has(runId)) { + return { + runId, + status: 'error', + stage: 'capturing', + done: true, + captured: { wiki: [], sl: [], xrefs: [] }, + error: `${runId} failed`, + commitHash: null, + skillsLoaded: [], + signalDetected: false, + } satisfies MemoryCaptureStatus; + } + return { + runId, + status: 'done', + stage: 'capturing', + done: true, + captured: { wiki: [`wiki-${runId}`], sl: [`sl-${runId}`], xrefs: [] }, + error: null, + commitHash: `commit-${runId}`, + skillsLoaded: ['wiki_capture', 'sl'], + signalDetected: true, + } satisfies MemoryCaptureStatus; + }), + }; +} + +function fakeProject(projectDir = '/tmp/project'): KtxLocalProject { + return { projectDir } as KtxLocalProject; +} + +describe('runKtxTextIngest', () => { + it('captures repeated inline text sequentially with generated internal chat ids', async () => { + const io = makeIo(); + const events: string[] = []; + const capture = fakeCapture({ events }); + const createMemoryCapture = vi.fn(() => capture); + + await expect( + runKtxTextIngest( + { + projectDir: '/tmp/project', + texts: ['Revenue means gross receipts.', 'Orders are completed purchases.'], + files: [], + userId: 'local-cli', + json: true, + failFast: false, + }, + io.io, + { + loadProject: vi.fn(async () => fakeProject()), + createMemoryCapture, + now: () => 1_700_000_000_000, + }, + ), + ).resolves.toBe(0); + + expect(createMemoryCapture).toHaveBeenCalledWith({ projectDir: '/tmp/project' }); + expect(capture.capture).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + userId: 'local-cli', + chatId: 'cli-text-ingest-1700000000000-1', + userMessage: 'Ingest external text artifact "Revenue means gross receipts." into KTX memory.', + assistantMessage: 'Revenue means gross receipts.', + sourceType: 'external_ingest', + }), + ); + expect(capture.capture).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + chatId: 'cli-text-ingest-1700000000000-2', + userMessage: 'Ingest external text artifact "Orders are completed purchases." into KTX memory.', + assistantMessage: 'Orders are completed purchases.', + }), + ); + expect(capture.capture).not.toHaveBeenCalledWith(expect.objectContaining({ connectionId: expect.anything() })); + expect(events).toEqual(['capture:run-1', 'wait:run-1', 'status:run-1', 'capture:run-2', 'wait:run-2', 'status:run-2']); + expect(JSON.parse(io.stdout())).toMatchObject({ + status: 'done', + results: [ + { + label: '"Revenue means gross receipts."', + runId: 'run-1', + status: 'done', + captured: { wiki: ['wiki-run-1'], sl: ['sl-run-1'] }, + }, + { + label: '"Orders are completed purchases."', + runId: 'run-2', + status: 'done', + captured: { wiki: ['wiki-run-2'], sl: ['sl-run-2'] }, + }, + ], + }); + }); + + it('loads files and stdin as batch items and passes a global connection id', async () => { + const io = makeIo(); + const capture = fakeCapture(); + + await expect( + runKtxTextIngest( + { + projectDir: '/tmp/project', + texts: [], + files: ['/tmp/docs/revenue.md', '-'], + connectionId: 'warehouse', + userId: 'agent', + json: false, + failFast: false, + }, + io.io, + { + loadProject: vi.fn(async () => fakeProject()), + createMemoryCapture: vi.fn(() => capture), + readFile: vi.fn(async (path) => `file:${path}`), + readStdin: vi.fn(async () => 'stdin content'), + now: () => 10, + }, + ), + ).resolves.toBe(0); + + expect(capture.capture).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + connectionId: 'warehouse', + userId: 'agent', + userMessage: 'Ingest external text artifact "revenue.md" into KTX memory.', + assistantMessage: 'file:/tmp/docs/revenue.md', + }), + ); + expect(capture.capture).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + connectionId: 'warehouse', + userMessage: 'Ingest external text artifact "stdin" into KTX memory.', + assistantMessage: 'stdin content', + }), + ); + expect(io.stdout()).toContain('Ingesting text memory'); + expect(io.stdout()).toContain('Texts:'); + expect(io.stdout()).toContain('revenue.md'); + expect(io.stdout()).toContain('stdin'); + }); + + it('uses bounded inline text previews as labels in plain output and capture metadata', async () => { + const io = makeIo(); + const capture = fakeCapture(); + const longText = `This inline note is intentionally long ${'x'.repeat(120)}`; + + await expect( + runKtxTextIngest( + { + projectDir: '/tmp/project', + texts: ['remember to call me Andrey', ' first line\n\tsecond line ', longText], + files: [], + userId: 'local-cli', + json: false, + failFast: false, + }, + io.io, + { + loadProject: vi.fn(async () => fakeProject()), + createMemoryCapture: vi.fn(() => capture), + now: () => 10, + }, + ), + ).resolves.toBe(0); + + const output = io.stdout(); + expect(output).toContain('"remember to call me Andrey"'); + expect(output).toContain('"first line second line"'); + expect(output).toContain('"This inline note is intentionally long xxxxxxxx..."'); + expect(output).not.toContain('text-1'); + expect(output).not.toContain(longText); + + expect(capture.capture).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + userMessage: 'Ingest external text artifact "remember to call me Andrey" into KTX memory.', + }), + ); + expect(capture.capture).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + userMessage: 'Ingest external text artifact "first line second line" into KTX memory.', + }), + ); + expect(capture.capture).toHaveBeenNthCalledWith( + 3, + expect.objectContaining({ + userMessage: 'Ingest external text artifact "This inline note is intentionally long xxxxxxxx..." into KTX memory.', + }), + ); + }); + + it('continues after an item failure by default and stops when failFast is set', async () => { + const continueIo = makeIo(); + const continueCapture = fakeCapture({ failRunIds: new Set(['run-1']) }); + + await expect( + runKtxTextIngest( + { + projectDir: '/tmp/project', + texts: ['bad', 'good'], + files: [], + userId: 'local-cli', + json: true, + failFast: false, + }, + continueIo.io, + { + loadProject: vi.fn(async () => fakeProject()), + createMemoryCapture: vi.fn(() => continueCapture), + }, + ), + ).resolves.toBe(1); + + expect(continueCapture.capture).toHaveBeenCalledTimes(2); + expect(JSON.parse(continueIo.stdout())).toMatchObject({ + status: 'failed', + results: [ + { label: '"bad"', status: 'error', error: 'run-1 failed' }, + { label: '"good"', status: 'done' }, + ], + }); + + const failFastIo = makeIo(); + const failFastCapture = fakeCapture({ failRunIds: new Set(['run-1']) }); + + await expect( + runKtxTextIngest( + { + projectDir: '/tmp/project', + texts: ['bad', 'skipped'], + files: [], + userId: 'local-cli', + json: true, + failFast: true, + }, + failFastIo.io, + { + loadProject: vi.fn(async () => fakeProject()), + createMemoryCapture: vi.fn(() => failFastCapture), + }, + ), + ).resolves.toBe(1); + + expect(failFastCapture.capture).toHaveBeenCalledTimes(1); + expect(JSON.parse(failFastIo.stdout()).results).toHaveLength(1); + }); + + it('rejects empty batches and empty text items', async () => { + const noInputIo = makeIo(); + await expect( + runKtxTextIngest( + { + projectDir: '/tmp/project', + texts: [], + files: [], + userId: 'local-cli', + json: false, + failFast: false, + }, + noInputIo.io, + { loadProject: vi.fn(), createMemoryCapture: vi.fn() }, + ), + ).resolves.toBe(1); + expect(noInputIo.stderr()).toContain('Provide at least one text item'); + + const emptyIo = makeIo(); + await expect( + runKtxTextIngest( + { + projectDir: '/tmp/project', + texts: [' '], + files: [], + userId: 'local-cli', + json: false, + failFast: false, + }, + emptyIo.io, + { loadProject: vi.fn(), createMemoryCapture: vi.fn() }, + ), + ).resolves.toBe(1); + expect(emptyIo.stderr()).toContain('Text item "text-1" is empty'); + }); +}); diff --git a/packages/cli/src/text-ingest.ts b/packages/cli/src/text-ingest.ts new file mode 100644 index 00000000..d48ee24b --- /dev/null +++ b/packages/cli/src/text-ingest.ts @@ -0,0 +1,354 @@ +import { readFile as fsReadFile } from 'node:fs/promises'; +import { basename, resolve } from 'node:path'; +import { createLocalProjectMemoryCapture, type MemoryAgentInput, type MemoryCaptureStatus } from '@ktx/context/memory'; +import { loadKtxProject, type KtxLocalProject } from '@ktx/context/project'; +import type { KtxCliIo } from './cli-runtime.js'; +import { createRepainter, initViewState, renderContextBuildView, type ContextBuildTargetState } from './context-build-view.js'; +import { formatDuration } from './demo-metrics.js'; +import type { KtxPublicIngestPlanTarget } from './public-ingest.js'; + +export interface KtxTextIngestArgs { + projectDir: string; + texts: string[]; + files: string[]; + connectionId?: string; + userId: string; + json: boolean; + failFast: boolean; +} + +export interface TextMemoryCapturePort { + capture(input: MemoryAgentInput): Promise<{ runId: string }>; + waitForRun(runId: string): Promise; + status(runId: string): Promise; +} + +interface TextIngestItem { + label: string; + content: string; +} + +interface TextIngestResult { + label: string; + runId: string | null; + status: 'done' | 'error'; + captured: MemoryCaptureStatus['captured']; + commitHash: string | null; + error: string | null; +} + +export interface KtxTextIngestDeps { + loadProject?: (options: { projectDir: string }) => Promise; + createMemoryCapture?: (project: KtxLocalProject) => TextMemoryCapturePort; + readFile?: (path: string) => Promise; + readStdin?: () => Promise; + now?: () => number; +} + +const INLINE_TEXT_LABEL_MAX_LENGTH = 50; +const ANSI_ESCAPE_PATTERN = /\x1B\[[0-?]*[ -/]*[@-~]/g; + +function defaultCreateMemoryCapture(project: KtxLocalProject): TextMemoryCapturePort { + return createLocalProjectMemoryCapture(project); +} + +async function defaultReadStdin(): Promise { + const chunks: string[] = []; + process.stdin.setEncoding('utf-8'); + for await (const chunk of process.stdin) { + chunks.push(String(chunk)); + } + return chunks.join(''); +} + +async function defaultReadFile(path: string): Promise { + return await fsReadFile(path, 'utf-8'); +} + +function emptyCaptured(): MemoryCaptureStatus['captured'] { + return { wiki: [], sl: [], xrefs: [] }; +} + +function normalizedTextPreview(content: string): string { + return content + .replace(ANSI_ESCAPE_PATTERN, '') + .replace(/[\u0000-\u001f\u007f-\u009f]/g, ' ') + .replace(/\s+/g, ' ') + .trim(); +} + +function truncateLabel(label: string, maxLength = INLINE_TEXT_LABEL_MAX_LENGTH): string { + const chars = Array.from(label); + if (chars.length <= maxLength) { + return label; + } + return `${chars.slice(0, maxLength - 3).join('').trimEnd()}...`; +} + +function quoteInlineTextLabel(label: string): string { + return JSON.stringify(label); +} + +function makeUniqueLabel(label: string, usedLabels: Set): string { + if (!usedLabels.has(label)) { + return label; + } + + for (let index = 2; ; index++) { + const suffix = ` (${index})`; + const candidate = `${truncateLabel(label, INLINE_TEXT_LABEL_MAX_LENGTH - suffix.length)}${suffix}`; + if (!usedLabels.has(candidate)) { + return candidate; + } + } +} + +function textLabel(content: string, index: number, usedLabels: Set): string { + const preview = normalizedTextPreview(content); + const baseLabel = preview.length > 0 ? quoteInlineTextLabel(truncateLabel(preview)) : `text-${index + 1}`; + return makeUniqueLabel(baseLabel, usedLabels); +} + +function artifactReference(label: string): string { + return label.startsWith('"') ? label : `"${label}"`; +} + +function stdinLabel(items: TextIngestItem[]): string { + if (!items.some((item) => item.label === 'stdin')) { + return 'stdin'; + } + return `stdin-${items.filter((item) => item.label.startsWith('stdin')).length + 1}`; +} + +async function loadItems(args: KtxTextIngestArgs, deps: KtxTextIngestDeps): Promise { + const items: TextIngestItem[] = []; + const usedTextLabels = new Set(); + args.texts.forEach((content, index) => { + const label = textLabel(content, index, usedTextLabels); + usedTextLabels.add(label); + items.push({ label, content }); + }); + + const readFile = deps.readFile ?? defaultReadFile; + const readStdin = deps.readStdin ?? defaultReadStdin; + for (const file of args.files) { + if (file === '-') { + items.push({ label: stdinLabel(items), content: await readStdin() }); + } else { + const path = resolve(file); + items.push({ label: basename(path), content: await readFile(path) }); + } + } + + return items; +} + +function validateItems(items: TextIngestItem[], io: KtxCliIo): boolean { + if (items.length === 0) { + io.stderr.write('Provide at least one text item with --text, a file path, or - for stdin.\n'); + return false; + } + + for (const item of items) { + if (item.content.trim().length === 0) { + io.stderr.write(`Text item "${item.label}" is empty.\n`); + return false; + } + } + return true; +} + +function makeTarget(label: string): KtxPublicIngestPlanTarget { + return { + connectionId: label, + driver: 'text', + operation: 'source-ingest', + debugCommand: '', + steps: ['memory-update'], + }; +} + +function allTargets(state: ReturnType): ContextBuildTargetState[] { + return [...state.primarySources, ...state.contextSources]; +} + +function renderTextIngestView(state: ReturnType, styled: boolean): string { + return renderContextBuildView(state, { + styled, + title: 'Ingesting text memory', + contextGroupLabel: 'Texts', + sourceIngestRunningText: 'capturing...', + completedItemName: { singular: 'text', plural: 'texts' }, + }); +} + +function summarizeCaptured(captured: MemoryCaptureStatus['captured']): string { + const parts = [ + `wiki=${captured.wiki.length}`, + `sl=${captured.sl.length}`, + `xrefs=${captured.xrefs.length}`, + ]; + return parts.join(', '); +} + +function resultFromStatus(label: string, status: MemoryCaptureStatus): TextIngestResult { + return { + label, + runId: status.runId, + status: status.status === 'done' ? 'done' : 'error', + captured: status.captured, + commitHash: status.commitHash, + error: status.error, + }; +} + +function errorResult(label: string, runId: string | null, error: unknown): TextIngestResult { + return { + label, + runId, + status: 'error', + captured: emptyCaptured(), + commitHash: null, + error: error instanceof Error ? error.message : String(error), + }; +} + +function writeJsonResult(args: KtxTextIngestArgs, results: TextIngestResult[], io: KtxCliIo): void { + io.stdout.write( + `${JSON.stringify( + { + status: results.some((result) => result.status === 'error') ? 'failed' : 'done', + projectDir: args.projectDir, + connectionId: args.connectionId ?? null, + results, + }, + null, + 2, + )}\n`, + ); +} + +function writePlainFailures(results: TextIngestResult[], io: KtxCliIo): void { + const failures = results.filter((result) => result.status === 'error'); + if (failures.length === 0) { + return; + } + + io.stdout.write('\nFailed text items:\n'); + for (const result of failures) { + io.stdout.write(` ${result.label}: ${result.error ?? 'failed'}\n`); + } +} + +export async function runKtxTextIngest( + args: KtxTextIngestArgs, + io: KtxCliIo, + deps: KtxTextIngestDeps = {}, +): Promise { + const items = await loadItems(args, deps); + if (!validateItems(items, io)) { + return 1; + } + + const project = await (deps.loadProject ?? loadKtxProject)({ projectDir: args.projectDir }); + const memoryCapture = (deps.createMemoryCapture ?? defaultCreateMemoryCapture)(project); + const now = deps.now ?? (() => Date.now()); + const batchId = now(); + const state = initViewState(items.map((item) => makeTarget(item.label))); + const targets = allTargets(state); + const isTTY = io.stdout.isTTY === true && args.json !== true; + const repainter = isTTY ? createRepainter(io) : null; + const results: TextIngestResult[] = []; + + state.startedAt = now(); + const paint = () => repainter?.paint(renderTextIngestView(state, true)); + paint(); + + let spinnerInterval: ReturnType | null = null; + if (repainter) { + spinnerInterval = setInterval(() => { + const current = now(); + state.frame++; + state.totalElapsedMs = state.startedAt === null ? 0 : current - state.startedAt; + for (const target of targets) { + if (target.status === 'running' && target.startedAt !== null) { + target.elapsedMs = current - target.startedAt; + } + } + paint(); + }, 140); + } + + try { + for (let index = 0; index < items.length; index++) { + const item = items[index]!; + const target = targets[index]!; + target.status = 'running'; + target.startedAt = now(); + target.detailLine = 'capturing...'; + target.progressUpdatedAtMs = target.startedAt; + paint(); + + let runId: string | null = null; + let result: TextIngestResult; + try { + const captureInput: MemoryAgentInput = { + userId: args.userId, + chatId: `cli-text-ingest-${batchId}-${index + 1}`, + userMessage: `Ingest external text artifact ${artifactReference(item.label)} into KTX memory.`, + assistantMessage: item.content.trim(), + ...(args.connectionId ? { connectionId: args.connectionId } : {}), + sourceType: 'external_ingest', + }; + const capture = await memoryCapture.capture(captureInput); + runId = capture.runId; + await memoryCapture.waitForRun(runId); + const status = await memoryCapture.status(runId); + if (!status) { + throw new Error(`Memory capture run "${runId}" was not found.`); + } + result = resultFromStatus(item.label, status); + } catch (error) { + result = errorResult(item.label, runId, error); + } + + results.push(result); + target.elapsedMs = now() - (target.startedAt ?? now()); + target.detailLine = null; + target.status = result.status === 'done' ? 'done' : 'failed'; + target.summaryText = result.status === 'done' ? summarizeCaptured(result.captured) : null; + target.failureText = result.status === 'error' ? result.error : null; + paint(); + + if (result.status === 'error' && args.failFast) { + break; + } + } + } finally { + if (spinnerInterval) { + clearInterval(spinnerInterval); + } + } + + if (state.startedAt !== null) { + state.totalElapsedMs = now() - state.startedAt; + } + + if (args.json) { + writeJsonResult(args, results, io); + } else if (repainter) { + repainter.paint(renderTextIngestView(state, true)); + writePlainFailures(results, io); + } else { + io.stdout.write(renderTextIngestView(state, false)); + writePlainFailures(results, io); + } + + if (!args.json && results.length > 0) { + const duration = state.totalElapsedMs > 0 ? ` in ${formatDuration(state.totalElapsedMs)}` : ''; + const outcome = results.some((result) => result.status === 'error') ? 'finished with failures' : 'finished'; + io.stdout.write(`Text memory ingest ${outcome}${duration}.\n`); + } + + return results.some((result) => result.status === 'error') ? 1 : 0; +} diff --git a/packages/connector-bigquery/src/connector.test.ts b/packages/connector-bigquery/src/connector.test.ts index a1c23864..46dc3b53 100644 --- a/packages/connector-bigquery/src/connector.test.ts +++ b/packages/connector-bigquery/src/connector.test.ts @@ -100,7 +100,6 @@ const connection = { dataset_id: 'analytics', credentials_json: JSON.stringify({ project_id: 'project-1', client_email: 'reader@example.test' }), location: 'US', - readonly: true, }; describe('KtxBigQueryScanConnector', () => { @@ -112,12 +111,6 @@ describe('KtxBigQueryScanConnector', () => { datasetIds: ['analytics'], location: 'US', }); - expect(() => - bigQueryConnectionConfigFromConfig({ - connectionId: 'warehouse', - connection: { ...connection, readonly: false }, - }), - ).toThrow('Native BigQuery connector requires connections.warehouse.readonly: true'); }); it('introspects datasets, table metadata, primary keys, and normalized types', async () => { diff --git a/packages/connector-bigquery/src/connector.ts b/packages/connector-bigquery/src/connector.ts index a994912e..72cb8129 100644 --- a/packages/connector-bigquery/src/connector.ts +++ b/packages/connector-bigquery/src/connector.ts @@ -30,7 +30,6 @@ export interface KtxBigQueryConnectionConfig { dataset_ids?: string[]; credentials_json?: string; location?: string; - readonly?: boolean; [key: string]: unknown; } @@ -194,7 +193,9 @@ function normalizeValue(value: unknown): unknown { return value; } -export function isKtxBigQueryConnectionConfig(connection: KtxBigQueryConnectionConfig | undefined): boolean { +export function isKtxBigQueryConnectionConfig( + connection: KtxBigQueryConnectionConfig | undefined, +): connection is KtxBigQueryConnectionConfig { return String(connection?.driver ?? '').toLowerCase() === 'bigquery'; } @@ -203,11 +204,9 @@ export function bigQueryConnectionConfigFromConfig(input: { connection: KtxBigQueryConnectionConfig | undefined; env?: NodeJS.ProcessEnv; }): KtxBigQueryResolvedConnectionConfig { + const inputDriver = input.connection?.driver ?? 'unknown'; if (!isKtxBigQueryConnectionConfig(input.connection)) { - throw new Error(`Native BigQuery connector cannot run driver "${input.connection?.driver ?? 'unknown'}"`); - } - if (input.connection?.readonly !== true) { - throw new Error(`Native BigQuery connector requires connections.${input.connectionId}.readonly: true`); + throw new Error(`Native BigQuery connector cannot run driver "${inputDriver}"`); } const env = input.env ?? process.env; diff --git a/packages/connector-clickhouse/src/connector.test.ts b/packages/connector-clickhouse/src/connector.test.ts index 7ed60efa..4f8b7f52 100644 --- a/packages/connector-clickhouse/src/connector.test.ts +++ b/packages/connector-clickhouse/src/connector.test.ts @@ -112,7 +112,6 @@ describe('KtxClickHouseScanConnector', () => { username: 'reader', password: 'test-pass', // pragma: allowlist secret ssl: true, - readonly: true, }, }), ).toMatchObject({ @@ -123,12 +122,6 @@ describe('KtxClickHouseScanConnector', () => { password: 'test-pass', // pragma: allowlist secret ssl: true, }); - expect(() => - clickHouseClientConfigFromConfig({ - connectionId: 'warehouse', - connection: { driver: 'clickhouse', host: 'ch.example.test', database: 'analytics', readonly: false }, - }), - ).toThrow('Native ClickHouse connector requires connections.warehouse.readonly: true'); }); it('introspects schema, primary keys, comments, row counts, and views', async () => { @@ -140,7 +133,6 @@ describe('KtxClickHouseScanConnector', () => { database: 'analytics', username: 'reader', password: 'test-pass', // pragma: allowlist secret - readonly: true, }, clientFactory: fakeClientFactory(), now: () => new Date('2026-04-29T14:00:00.000Z'), @@ -189,7 +181,6 @@ describe('KtxClickHouseScanConnector', () => { database: 'analytics', username: 'reader', password: 'test-pass', // pragma: allowlist secret - readonly: true, }, clientFactory, }); @@ -253,7 +244,6 @@ describe('KtxClickHouseScanConnector', () => { database: 'analytics', username: 'reader', password: 'test-pass', // pragma: allowlist secret - readonly: true, }, }, clientFactory: fakeClientFactory(), diff --git a/packages/connector-clickhouse/src/connector.ts b/packages/connector-clickhouse/src/connector.ts index 0273a62b..4b39c943 100644 --- a/packages/connector-clickhouse/src/connector.ts +++ b/packages/connector-clickhouse/src/connector.ts @@ -35,7 +35,6 @@ export interface KtxClickHouseConnectionConfig { password?: string; url?: string; ssl?: boolean; - readonly?: boolean; [key: string]: unknown; } @@ -193,7 +192,9 @@ function isNullableClickHouseType(type: string): boolean { return type.startsWith('Nullable(') || type.startsWith('LowCardinality(Nullable('); } -export function isKtxClickHouseConnectionConfig(connection: KtxClickHouseConnectionConfig | undefined): boolean { +export function isKtxClickHouseConnectionConfig( + connection: KtxClickHouseConnectionConfig | undefined, +): connection is KtxClickHouseConnectionConfig { return String(connection?.driver ?? '').toLowerCase() === 'clickhouse'; } @@ -202,11 +203,9 @@ export function clickHouseClientConfigFromConfig(input: { connection: KtxClickHouseConnectionConfig | undefined; env?: NodeJS.ProcessEnv; }): KtxClickHouseResolvedClientConfig { + const inputDriver = input.connection?.driver ?? 'unknown'; if (!isKtxClickHouseConnectionConfig(input.connection)) { - throw new Error(`Native ClickHouse connector cannot run driver "${input.connection?.driver ?? 'unknown'}"`); - } - if (input.connection?.readonly !== true) { - throw new Error(`Native ClickHouse connector requires connections.${input.connectionId}.readonly: true`); + throw new Error(`Native ClickHouse connector cannot run driver "${inputDriver}"`); } const env = input.env ?? process.env; diff --git a/packages/connector-mysql/src/connector.test.ts b/packages/connector-mysql/src/connector.test.ts index 3e7ac1e1..c5c5a3fa 100644 --- a/packages/connector-mysql/src/connector.test.ts +++ b/packages/connector-mysql/src/connector.test.ts @@ -92,7 +92,7 @@ function fakePoolFactory(): KtxMysqlPoolFactory { describe('KtxMysqlScanConnector', () => { it('resolves MySQL connection configuration safely', () => { - expect(isKtxMysqlConnectionConfig({ driver: 'mysql', host: 'localhost', database: 'analytics', readonly: true })).toBe(true); + expect(isKtxMysqlConnectionConfig({ driver: 'mysql', host: 'localhost', database: 'analytics' })).toBe(true); expect(isKtxMysqlConnectionConfig({ driver: 'postgres', host: 'localhost', database: 'analytics' })).toBe(false); expect( mysqlConnectionPoolConfigFromConfig({ @@ -105,7 +105,6 @@ describe('KtxMysqlScanConnector', () => { username: 'reader', password: 'secret', // pragma: allowlist secret ssl: true, - readonly: true, }, }), ).toMatchObject({ @@ -116,12 +115,6 @@ describe('KtxMysqlScanConnector', () => { password: 'secret', // pragma: allowlist secret ssl: { rejectUnauthorized: false }, }); - expect(() => - mysqlConnectionPoolConfigFromConfig({ - connectionId: 'warehouse', - connection: { driver: 'mysql', host: 'db.example.test', database: 'analytics', readonly: false }, - }), - ).toThrow('Native MySQL connector requires connections.warehouse.readonly: true'); }); it('introspects schema, primary keys, comments, row counts, views, and foreign keys', async () => { @@ -133,7 +126,6 @@ describe('KtxMysqlScanConnector', () => { database: 'analytics', username: 'reader', password: 'secret', // pragma: allowlist secret - readonly: true, }, poolFactory: fakePoolFactory(), now: () => new Date('2026-04-29T12:00:00.000Z'), @@ -192,7 +184,6 @@ describe('KtxMysqlScanConnector', () => { database: 'analytics', username: 'reader', password: 'secret', // pragma: allowlist secret - readonly: true, }, poolFactory, }); @@ -249,7 +240,6 @@ describe('KtxMysqlScanConnector', () => { database: 'analytics', username: 'reader', password: 'secret', // pragma: allowlist secret - readonly: true, }, }, poolFactory: fakePoolFactory(), diff --git a/packages/connector-mysql/src/connector.ts b/packages/connector-mysql/src/connector.ts index 69a09272..62bb1880 100644 --- a/packages/connector-mysql/src/connector.ts +++ b/packages/connector-mysql/src/connector.ts @@ -35,7 +35,6 @@ export interface KtxMysqlConnectionConfig { password?: string; url?: string; ssl?: boolean | { rejectUnauthorized?: boolean }; - readonly?: boolean; [key: string]: unknown; } @@ -232,7 +231,9 @@ function queryParams(params: Record | unknown[] | undefined): u return Array.isArray(params) ? params : Object.values(params); } -export function isKtxMysqlConnectionConfig(connection: KtxMysqlConnectionConfig | undefined): boolean { +export function isKtxMysqlConnectionConfig( + connection: KtxMysqlConnectionConfig | undefined, +): connection is KtxMysqlConnectionConfig { return String(connection?.driver ?? '').toLowerCase() === 'mysql'; } @@ -241,11 +242,9 @@ export function mysqlConnectionPoolConfigFromConfig(input: { connection: KtxMysqlConnectionConfig | undefined; env?: NodeJS.ProcessEnv; }): KtxMysqlPoolConfig { + const inputDriver = input.connection?.driver ?? 'unknown'; if (!isKtxMysqlConnectionConfig(input.connection)) { - throw new Error(`Native MySQL connector cannot run driver "${input.connection?.driver ?? 'unknown'}"`); - } - if (input.connection?.readonly !== true) { - throw new Error(`Native MySQL connector requires connections.${input.connectionId}.readonly: true`); + throw new Error(`Native MySQL connector cannot run driver "${inputDriver}"`); } const env = input.env ?? process.env; diff --git a/packages/connector-postgres/src/connector.test.ts b/packages/connector-postgres/src/connector.test.ts index 96443c90..8093acda 100644 --- a/packages/connector-postgres/src/connector.test.ts +++ b/packages/connector-postgres/src/connector.test.ts @@ -102,7 +102,7 @@ function metadataResults(): Map { describe('KtxPostgresScanConnector', () => { it('resolves configuration safely', () => { - expect(isKtxPostgresConnectionConfig({ driver: 'postgres', url: 'env:DATABASE_URL', readonly: true })).toBe(true); + expect(isKtxPostgresConnectionConfig({ driver: 'postgres', url: 'env:DATABASE_URL' })).toBe(true); expect(isKtxPostgresConnectionConfig({ driver: 'postgresql', host: 'db', database: 'analytics' })).toBe(true); expect(isKtxPostgresConnectionConfig({ driver: 'mysql', host: 'db' })).toBe(false); expect( @@ -115,7 +115,6 @@ describe('KtxPostgresScanConnector', () => { username: 'reader', password: 'test-password', // pragma: allowlist secret schemas: ['analytics', 'public'], - readonly: true, ssl: true, rejectUnauthorized: false, }, @@ -134,7 +133,6 @@ describe('KtxPostgresScanConnector', () => { connection: { driver: 'postgres', url: 'env:DEMO_DATABASE_URL', - readonly: true, }, env: { DEMO_DATABASE_URL: 'postgresql://reader@demo.example.test:5432/demo?sslmode=prefer', @@ -148,12 +146,16 @@ describe('KtxPostgresScanConnector', () => { }); expect(libpqPreferConfig).not.toHaveProperty('connectionString'); expect(libpqPreferConfig).not.toHaveProperty('ssl'); - expect(() => + expect( postgresPoolConfigFromConfig({ connectionId: 'warehouse', connection: { driver: 'postgres', host: 'db.example.test', database: 'analytics', username: 'reader' }, }), - ).toThrow('Native PostgreSQL connector requires connections.warehouse.readonly: true'); + ).toMatchObject({ + host: 'db.example.test', + database: 'analytics', + user: 'reader', + }); }); it('introspects schemas, tables, views, primary keys, comments, row counts, and foreign keys', async () => { @@ -166,7 +168,6 @@ describe('KtxPostgresScanConnector', () => { username: 'reader', password: 'test-password', // pragma: allowlist secret schema: 'public', - readonly: true, }, poolFactory: fakePoolFactory(metadataResults()), now: () => new Date('2026-04-29T10:00:00.000Z'), @@ -225,7 +226,6 @@ describe('KtxPostgresScanConnector', () => { username: 'reader', password: 'test-password', // pragma: allowlist secret schema: 'public', - readonly: true, }, poolFactory: fakePoolFactory(metadataResults()), }); @@ -274,7 +274,6 @@ describe('KtxPostgresScanConnector', () => { username: 'reader', password: 'test-password', // pragma: allowlist secret schema: 'public', - readonly: true, }, }, poolFactory: fakePoolFactory(metadataResults()), @@ -347,7 +346,6 @@ describe('KtxPostgresScanConnector', () => { username: 'reader', password: 'test-password', // pragma: allowlist secret schema: 'public', - readonly: true, }, }, poolFactory: endAwarePoolFactory, @@ -383,7 +381,6 @@ describe('KtxPostgresScanConnector', () => { database: 'analytics', username: 'reader', password: 'test-password', // pragma: allowlist secret - readonly: true, }, poolFactory, }); diff --git a/packages/connector-postgres/src/connector.ts b/packages/connector-postgres/src/connector.ts index 65490040..7f5ed65c 100644 --- a/packages/connector-postgres/src/connector.ts +++ b/packages/connector-postgres/src/connector.ts @@ -61,7 +61,6 @@ export interface KtxPostgresConnectionConfig { sslmode?: string; sslMode?: string; rejectUnauthorized?: boolean; - readonly?: boolean; [key: string]: unknown; } @@ -291,7 +290,9 @@ function searchPathSchemasFromConnection(connection: KtxPostgresConnectionConfig return schemas.includes('public') ? schemas : [...schemas, 'public']; } -export function isKtxPostgresConnectionConfig(connection: KtxPostgresConnectionConfig | undefined): boolean { +export function isKtxPostgresConnectionConfig( + connection: KtxPostgresConnectionConfig | undefined, +): connection is KtxPostgresConnectionConfig { const driver = String(connection?.driver ?? '').toLowerCase(); return driver === 'postgres' || driver === 'postgresql'; } @@ -301,11 +302,9 @@ export function postgresPoolConfigFromConfig(input: { connection: KtxPostgresConnectionConfig | undefined; env?: NodeJS.ProcessEnv; }): KtxPostgresPoolConfig { + const inputDriver = input.connection?.driver ?? 'unknown'; if (!isKtxPostgresConnectionConfig(input.connection)) { - throw new Error(`Native PostgreSQL connector cannot run driver "${input.connection?.driver ?? 'unknown'}"`); - } - if (input.connection?.readonly !== true) { - throw new Error(`Native PostgreSQL connector requires connections.${input.connectionId}.readonly: true`); + throw new Error(`Native PostgreSQL connector cannot run driver "${inputDriver}"`); } const env = input.env ?? process.env; diff --git a/packages/connector-postgres/src/historic-sql-query-client.test.ts b/packages/connector-postgres/src/historic-sql-query-client.test.ts index f02cb9c3..b9c9fd40 100644 --- a/packages/connector-postgres/src/historic-sql-query-client.test.ts +++ b/packages/connector-postgres/src/historic-sql-query-client.test.ts @@ -30,7 +30,6 @@ describe('KtxPostgresHistoricSqlQueryClient', () => { connectionId: 'warehouse', connection: { driver: 'postgres', - readonly: true, url: 'postgresql://readonly:secret@pg.example.test/warehouse', // pragma: allowlist secret }, poolFactory, diff --git a/packages/connector-snowflake/src/connector.test.ts b/packages/connector-snowflake/src/connector.test.ts index 91bb33d4..a49be885 100644 --- a/packages/connector-snowflake/src/connector.test.ts +++ b/packages/connector-snowflake/src/connector.test.ts @@ -78,7 +78,6 @@ describe('KtxSnowflakeScanConnector', () => { warehouse: 'WH', database: 'ANALYTICS', username: 'reader', - readonly: true, }), ).toBe(true); expect(isKtxSnowflakeConnectionConfig({ driver: 'bigquery' })).toBe(false); @@ -94,7 +93,6 @@ describe('KtxSnowflakeScanConnector', () => { schema_name: 'PUBLIC', username: 'reader', password: 'fixture-pass', // pragma: allowlist secret - readonly: true, }, }), ).toMatchObject({ @@ -105,12 +103,6 @@ describe('KtxSnowflakeScanConnector', () => { username: 'reader', authMethod: 'password', }); - expect(() => - snowflakeConnectionConfigFromConfig({ - connectionId: 'warehouse', - connection: { driver: 'snowflake', account: 'acct', readonly: false }, - }), - ).toThrow('Native Snowflake connector requires connections.warehouse.readonly: true'); }); it('introspects schema, primary keys, comments, row counts, and dimensions', async () => { @@ -125,7 +117,6 @@ describe('KtxSnowflakeScanConnector', () => { schema_name: 'PUBLIC', username: 'reader', password: 'fixture-pass', // pragma: allowlist secret - readonly: true, }, driverFactory: fakeDriverFactory(), now: () => new Date('2026-04-29T18:00:00.000Z'), @@ -185,7 +176,6 @@ describe('KtxSnowflakeScanConnector', () => { schema_name: 'PUBLIC', username: 'reader', password: 'fixture-pass', // pragma: allowlist secret - readonly: true, }, driverFactory, }); @@ -243,7 +233,6 @@ describe('KtxSnowflakeScanConnector', () => { schema_name: 'PUBLIC', username: 'reader', password: 'fixture-pass', // pragma: allowlist secret - readonly: true, }, }, driverFactory: fakeDriverFactory(), diff --git a/packages/connector-snowflake/src/connector.ts b/packages/connector-snowflake/src/connector.ts index 063976f7..76fc34fd 100644 --- a/packages/connector-snowflake/src/connector.ts +++ b/packages/connector-snowflake/src/connector.ts @@ -38,7 +38,6 @@ export interface KtxSnowflakeConnectionConfig { privateKey?: string; passphrase?: string; role?: string; - readonly?: boolean; [key: string]: unknown; } @@ -191,7 +190,9 @@ function toSnowflakeBinds(params: unknown[] | undefined): snowflake.Binds | unde return params?.map((value) => toSnowflakeBind(value)); } -export function isKtxSnowflakeConnectionConfig(connection: KtxSnowflakeConnectionConfig | undefined): boolean { +export function isKtxSnowflakeConnectionConfig( + connection: KtxSnowflakeConnectionConfig | undefined, +): connection is KtxSnowflakeConnectionConfig { return String(connection?.driver ?? '').toLowerCase() === 'snowflake'; } @@ -200,11 +201,9 @@ export function snowflakeConnectionConfigFromConfig(input: { connection: KtxSnowflakeConnectionConfig | undefined; env?: NodeJS.ProcessEnv; }): KtxSnowflakeResolvedConnectionConfig { + const inputDriver = input.connection?.driver ?? 'unknown'; if (!isKtxSnowflakeConnectionConfig(input.connection)) { - throw new Error(`Native Snowflake connector cannot run driver "${input.connection?.driver ?? 'unknown'}"`); - } - if (input.connection?.readonly !== true) { - throw new Error(`Native Snowflake connector requires connections.${input.connectionId}.readonly: true`); + throw new Error(`Native Snowflake connector cannot run driver "${inputDriver}"`); } const env = input.env ?? process.env; const authMethod = input.connection?.authMethod ?? 'password'; @@ -395,7 +394,7 @@ class SnowflakeSdkDriver implements KtxSnowflakeDriver { private async createConnection(): Promise { const patch = await this.sdkOptionsProvider?.resolve({ account: this.resolved.account, - connection: { ...this.resolved, driver: 'snowflake', readonly: true }, + connection: { ...this.resolved, driver: 'snowflake' }, }); if (patch?.close) { this.closeSdkOptions.push(patch.close); diff --git a/packages/connector-sqlite/src/connector.test.ts b/packages/connector-sqlite/src/connector.test.ts index 4bc26ec9..9bee5567 100644 --- a/packages/connector-sqlite/src/connector.test.ts +++ b/packages/connector-sqlite/src/connector.test.ts @@ -53,45 +53,43 @@ describe('KtxSqliteScanConnector', () => { writeFileSync(pointerPath, dbPath, 'utf-8'); try { - expect(isKtxSqliteConnectionConfig({ driver: 'sqlite', path: 'warehouse.db', readonly: true })).toBe(true); - expect(isKtxSqliteConnectionConfig({ driver: 'postgres', url: 'env:DATABASE_URL', readonly: true })).toBe( - false, - ); + expect(isKtxSqliteConnectionConfig({ driver: 'sqlite', path: 'warehouse.db' })).toBe(true); + expect(isKtxSqliteConnectionConfig({ driver: 'postgres', url: 'env:DATABASE_URL' })).toBe(false); expect( sqliteDatabasePathFromConfig({ connectionId: 'warehouse', projectDir: tempDir, - connection: { driver: 'sqlite', path: 'warehouse.db', readonly: true }, + connection: { driver: 'sqlite', path: 'warehouse.db' }, }), ).toBe(dbPath); expect( sqliteDatabasePathFromConfig({ connectionId: 'warehouse', projectDir: tempDir, - connection: { driver: 'sqlite', url: 'env:KTX_SQLITE_TEST_URL', readonly: true }, + connection: { driver: 'sqlite', url: 'env:KTX_SQLITE_TEST_URL' }, }), ).toBe(dbPath); expect( sqliteDatabasePathFromConfig({ connectionId: 'warehouse', projectDir: tempDir, - connection: { driver: 'sqlite', url: `file://${dbPath}`, readonly: true }, + connection: { driver: 'sqlite', url: `file://${dbPath}` }, }), ).toBe(dbPath); expect( sqliteDatabasePathFromConfig({ connectionId: 'warehouse', projectDir: tempDir, - connection: { driver: 'sqlite', path: `file:${pointerPath}`, readonly: true }, + connection: { driver: 'sqlite', path: `file:${pointerPath}` }, }), ).toBe(dbPath); - expect(() => + expect( sqliteDatabasePathFromConfig({ connectionId: 'warehouse', projectDir: tempDir, - connection: { driver: 'sqlite', path: 'warehouse.db', readonly: false }, + connection: { driver: 'sqlite', path: 'warehouse.db' }, }), - ).toThrow('Native SQLite connector requires connections.warehouse.readonly: true'); + ).toBe(dbPath); } finally { if (originalDatabaseUrl === undefined) { delete process.env.KTX_SQLITE_TEST_URL; @@ -104,7 +102,7 @@ describe('KtxSqliteScanConnector', () => { it('introspects schema, primary keys, row counts, views, and foreign keys', async () => { const connector = new KtxSqliteScanConnector({ connectionId: 'warehouse', - connection: { driver: 'sqlite', path: dbPath, readonly: true }, + connection: { driver: 'sqlite', path: dbPath }, now: () => new Date('2026-04-29T10:00:00.000Z'), }); @@ -151,7 +149,7 @@ describe('KtxSqliteScanConnector', () => { it('runs samples, distinct values, statistics, and read-only SQL', async () => { const connector = new KtxSqliteScanConnector({ connectionId: 'warehouse', - connection: { driver: 'sqlite', path: dbPath, readonly: true }, + connection: { driver: 'sqlite', path: dbPath }, }); await expect( @@ -199,7 +197,7 @@ describe('KtxSqliteScanConnector', () => { const introspection = createSqliteLiveDatabaseIntrospection({ projectDir: tempDir, connections: { - warehouse: { driver: 'sqlite', path: 'warehouse.db', readonly: true }, + warehouse: { driver: 'sqlite', path: 'warehouse.db' }, }, now: () => new Date('2026-04-29T10:00:00.000Z'), }); diff --git a/packages/connector-sqlite/src/connector.ts b/packages/connector-sqlite/src/connector.ts index c42db002..2a0d997e 100644 --- a/packages/connector-sqlite/src/connector.ts +++ b/packages/connector-sqlite/src/connector.ts @@ -29,7 +29,6 @@ export interface KtxSqliteConnectionConfig { path?: string; url?: string; file_path?: string; - readonly?: boolean; [key: string]: unknown; } @@ -135,17 +134,17 @@ function stripLeadingSqlComments(sql: string): string { return sql.slice(index); } -export function isKtxSqliteConnectionConfig(connection: KtxSqliteConnectionConfig | undefined): boolean { +export function isKtxSqliteConnectionConfig( + connection: KtxSqliteConnectionConfig | undefined, +): connection is KtxSqliteConnectionConfig { const driver = String(connection?.driver ?? '').toLowerCase(); return driver === 'sqlite' || driver === 'sqlite3'; } export function sqliteDatabasePathFromConfig(input: SqliteDatabasePathInput): string { + const inputDriver = input.connection?.driver ?? 'unknown'; if (!isKtxSqliteConnectionConfig(input.connection)) { - throw new Error(`Native SQLite connector cannot run driver "${input.connection?.driver ?? 'unknown'}"`); - } - if (input.connection?.readonly !== true) { - throw new Error(`Native SQLite connector requires connections.${input.connectionId}.readonly: true`); + throw new Error(`Native SQLite connector cannot run driver "${inputDriver}"`); } const configuredPath = stringConfigValue(input.connection, 'path') ?? diff --git a/packages/connector-sqlserver/src/connector.test.ts b/packages/connector-sqlserver/src/connector.test.ts index eebab0ba..b7915fa8 100644 --- a/packages/connector-sqlserver/src/connector.test.ts +++ b/packages/connector-sqlserver/src/connector.test.ts @@ -145,7 +145,6 @@ describe('KtxSqlServerScanConnector', () => { driver: 'sqlserver', host: 'localhost', database: 'analytics', - readonly: true, }), ).toBe(true); expect(isKtxSqlServerConnectionConfig({ driver: 'mysql', host: 'localhost', database: 'analytics' })).toBe(false); @@ -159,7 +158,6 @@ describe('KtxSqlServerScanConnector', () => { database: 'analytics', username: 'reader', trustServerCertificate: false, - readonly: true, }, }), ).toMatchObject({ @@ -169,12 +167,6 @@ describe('KtxSqlServerScanConnector', () => { user: 'reader', options: { encrypt: true, trustServerCertificate: false }, }); - expect(() => - sqlServerConnectionPoolConfigFromConfig({ - connectionId: 'warehouse', - connection: { driver: 'sqlserver', host: 'db.example.test', database: 'analytics', readonly: false }, - }), - ).toThrow('Native SQL Server connector requires connections.warehouse.readonly: true'); }); it('introspects schema, primary keys, comments, row counts, views, and foreign keys', async () => { @@ -186,7 +178,6 @@ describe('KtxSqlServerScanConnector', () => { database: 'analytics', username: 'reader', schema: 'dbo', - readonly: true, }, poolFactory: fakePoolFactory(), now: () => new Date('2026-04-29T16:00:00.000Z'), @@ -246,7 +237,6 @@ describe('KtxSqlServerScanConnector', () => { database: 'analytics', username: 'reader', schema: 'dbo', - readonly: true, }, poolFactory, }); @@ -315,7 +305,6 @@ describe('KtxSqlServerScanConnector', () => { database: 'analytics', username: 'reader', schema: 'dbo', - readonly: true, }, }, poolFactory: fakePoolFactory(), diff --git a/packages/connector-sqlserver/src/connector.ts b/packages/connector-sqlserver/src/connector.ts index 189ff98b..73a46aab 100644 --- a/packages/connector-sqlserver/src/connector.ts +++ b/packages/connector-sqlserver/src/connector.ts @@ -37,7 +37,6 @@ export interface KtxSqlServerConnectionConfig { schema?: string; schemas?: string[]; trustServerCertificate?: boolean; - readonly?: boolean; [key: string]: unknown; } @@ -234,7 +233,9 @@ function limitSqlForSqlServerExecution(sqlText: string, maxRows: number | undefi return `SELECT TOP ${maxRows} * FROM (${trimmed}) AS ktx_query_result`; } -export function isKtxSqlServerConnectionConfig(connection: KtxSqlServerConnectionConfig | undefined): boolean { +export function isKtxSqlServerConnectionConfig( + connection: KtxSqlServerConnectionConfig | undefined, +): connection is KtxSqlServerConnectionConfig { return String(connection?.driver ?? '').toLowerCase() === 'sqlserver'; } @@ -243,11 +244,9 @@ export function sqlServerConnectionPoolConfigFromConfig(input: { connection: KtxSqlServerConnectionConfig | undefined; env?: NodeJS.ProcessEnv; }): KtxSqlServerPoolConfig { + const inputDriver = input.connection?.driver ?? 'unknown'; if (!isKtxSqlServerConnectionConfig(input.connection)) { - throw new Error(`Native SQL Server connector cannot run driver "${input.connection?.driver ?? 'unknown'}"`); - } - if (input.connection?.readonly !== true) { - throw new Error(`Native SQL Server connector requires connections.${input.connectionId}.readonly: true`); + throw new Error(`Native SQL Server connector cannot run driver "${inputDriver}"`); } const env = input.env ?? process.env; diff --git a/packages/context/src/connections/local-query-executor.test.ts b/packages/context/src/connections/local-query-executor.test.ts index fd94c6dc..d2f77975 100644 --- a/packages/context/src/connections/local-query-executor.test.ts +++ b/packages/context/src/connections/local-query-executor.test.ts @@ -26,14 +26,14 @@ describe('createDefaultLocalQueryExecutor', () => { await expect( executor.execute({ connectionId: 'pg', - connection: { driver: 'postgres', readonly: true }, + connection: { driver: 'postgres' }, sql: 'select 1', }), ).resolves.toMatchObject({ headers: ['pg'] }); await expect( executor.execute({ connectionId: 'local', - connection: { driver: 'sqlite', readonly: true }, + connection: { driver: 'sqlite' }, sql: 'select 1', }), ).resolves.toMatchObject({ headers: ['sqlite'] }); @@ -51,7 +51,7 @@ describe('createDefaultLocalQueryExecutor', () => { await expect( executor.execute({ connectionId: 'warehouse', - connection: { driver: 'snowflake', readonly: true }, + connection: { driver: 'snowflake' }, sql: 'select 1', }), ).rejects.toThrow('No local query executor is configured for driver "snowflake".'); diff --git a/packages/context/src/connections/postgres-query-executor.test.ts b/packages/context/src/connections/postgres-query-executor.test.ts index 2c52d97e..6bb522cf 100644 --- a/packages/context/src/connections/postgres-query-executor.test.ts +++ b/packages/context/src/connections/postgres-query-executor.test.ts @@ -37,7 +37,7 @@ describe('createPostgresQueryExecutor', () => { const result = await executor.execute({ connectionId: 'warehouse', - connection: { driver: 'postgres', url: 'postgres://example/db', readonly: true }, + connection: { driver: 'postgres', url: 'postgres://example/db' }, sql: 'select status, count(*) as order_count from public.orders group by status', maxRows: 50, }); @@ -80,7 +80,7 @@ describe('createPostgresQueryExecutor', () => { await expect( executor.execute({ connectionId: 'warehouse', - connection: { driver: 'postgres', url: 'postgres://example/db', readonly: true }, + connection: { driver: 'postgres', url: 'postgres://example/db' }, sql: 'select * from broken', maxRows: 10, }), @@ -89,23 +89,15 @@ describe('createPostgresQueryExecutor', () => { expect(client.end).toHaveBeenCalledTimes(1); }); - it('requires a Postgres url and read-only connection config', async () => { + it('requires a Postgres url', async () => { const executor = createPostgresQueryExecutor({ clientFactory: vi.fn() }); await expect( executor.execute({ connectionId: 'warehouse', - connection: { driver: 'postgres', readonly: true }, + connection: { driver: 'postgres' }, sql: 'select 1', }), ).rejects.toThrow('Local Postgres execution requires connections.warehouse.url'); - - await expect( - executor.execute({ - connectionId: 'warehouse', - connection: { driver: 'postgres', url: 'postgres://example/db', readonly: false }, - sql: 'select 1', - }), - ).rejects.toThrow('Local query execution requires connections.warehouse.readonly: true'); }); }); diff --git a/packages/context/src/connections/postgres-query-executor.ts b/packages/context/src/connections/postgres-query-executor.ts index 2ab142a5..b5f2d02e 100644 --- a/packages/context/src/connections/postgres-query-executor.ts +++ b/packages/context/src/connections/postgres-query-executor.ts @@ -37,18 +37,16 @@ export function createPostgresQueryExecutor(options: PostgresQueryExecutorOption return { async execute(input: KtxSqlQueryExecutionInput): Promise { const driver = connectionDriver(input); + const connection = input.connection; if (driver !== 'postgres' && driver !== 'postgresql') { - throw new Error(`Local Postgres execution cannot run driver "${input.connection?.driver ?? 'unknown'}".`); + throw new Error(`Local Postgres execution cannot run driver "${connection?.driver ?? 'unknown'}".`); } - if (input.connection?.readonly !== true) { - throw new Error(`Local query execution requires connections.${input.connectionId}.readonly: true.`); - } - if (typeof input.connection.url !== 'string' || input.connection.url.trim().length === 0) { + if (typeof connection?.url !== 'string' || connection.url.trim().length === 0) { throw new Error(`Local Postgres execution requires connections.${input.connectionId}.url.`); } const client = clientFactory({ - connectionString: input.connection.url, + connectionString: connection.url, statement_timeout: options.statementTimeoutMs ?? 30_000, query_timeout: options.queryTimeoutMs ?? 35_000, connectionTimeoutMillis: options.connectionTimeoutMs ?? 5_000, diff --git a/packages/context/src/connections/sqlite-query-executor.test.ts b/packages/context/src/connections/sqlite-query-executor.test.ts index 3046f9bb..facb5139 100644 --- a/packages/context/src/connections/sqlite-query-executor.test.ts +++ b/packages/context/src/connections/sqlite-query-executor.test.ts @@ -38,7 +38,7 @@ describe('createSqliteQueryExecutor', () => { const result = await executor.execute({ connectionId: 'warehouse', projectDir: tempDir, - connection: { driver: 'sqlite', path: 'warehouse.db', readonly: true }, + connection: { driver: 'sqlite', path: 'warehouse.db' }, sql: 'select status, count(*) as order_count from orders group by status order by status', maxRows: 10, }); @@ -60,7 +60,7 @@ describe('createSqliteQueryExecutor', () => { sqliteDatabasePathFromConnection({ connectionId: 'warehouse', projectDir: tempDir, - connection: { driver: 'sqlite', url: `file://${dbPath}`, readonly: true }, + connection: { driver: 'sqlite', url: `file://${dbPath}` }, sql: 'select 1', }), ).toBe(dbPath); @@ -74,7 +74,7 @@ describe('createSqliteQueryExecutor', () => { sqliteDatabasePathFromConnection({ connectionId: 'warehouse', projectDir: tempDir, - connection: { driver: 'sqlite', path: `file:${pointerPath}`, readonly: true }, + connection: { driver: 'sqlite', path: `file:${pointerPath}` }, sql: 'select 1', }), ).toBe(dbPath); @@ -89,7 +89,7 @@ describe('createSqliteQueryExecutor', () => { sqliteDatabasePathFromConnection({ connectionId: 'warehouse', projectDir: tempDir, - connection: { driver: 'sqlite', url: 'env:KTX_SQLITE_TEST_URL', readonly: true }, + connection: { driver: 'sqlite', url: 'env:KTX_SQLITE_TEST_URL' }, sql: 'select 1', }), ).toBe(dbPath); @@ -109,20 +109,20 @@ describe('createSqliteQueryExecutor', () => { executor.execute({ connectionId: 'warehouse', projectDir: tempDir, - connection: { driver: 'sqlite', path: 'warehouse.db', readonly: true }, + connection: { driver: 'sqlite', path: 'warehouse.db' }, sql: 'delete from orders', }), ).rejects.toThrow('Only read-only SELECT/WITH queries can be executed locally'); }); - it('requires a SQLite driver, read-only config, and a database path', async () => { + it('requires a SQLite driver and a database path', async () => { const executor = createSqliteQueryExecutor(); await expect( executor.execute({ connectionId: 'warehouse', projectDir: tempDir, - connection: { driver: 'postgres', path: 'warehouse.db', readonly: true }, + connection: { driver: 'postgres', path: 'warehouse.db' }, sql: 'select 1', }), ).rejects.toThrow('Local SQLite execution cannot run driver "postgres"'); @@ -131,16 +131,7 @@ describe('createSqliteQueryExecutor', () => { executor.execute({ connectionId: 'warehouse', projectDir: tempDir, - connection: { driver: 'sqlite', path: 'warehouse.db', readonly: false }, - sql: 'select 1', - }), - ).rejects.toThrow('Local query execution requires connections.warehouse.readonly: true'); - - await expect( - executor.execute({ - connectionId: 'warehouse', - projectDir: tempDir, - connection: { driver: 'sqlite', readonly: true }, + connection: { driver: 'sqlite' }, sql: 'select 1', }), ).rejects.toThrow('Local SQLite execution requires connections.warehouse.path or connections.warehouse.url'); diff --git a/packages/context/src/connections/sqlite-query-executor.ts b/packages/context/src/connections/sqlite-query-executor.ts index d32a37ba..2a87ef7d 100644 --- a/packages/context/src/connections/sqlite-query-executor.ts +++ b/packages/context/src/connections/sqlite-query-executor.ts @@ -54,9 +54,6 @@ export function sqliteDatabasePathFromConnection(input: KtxSqlQueryExecutionInpu if (driver !== 'sqlite' && driver !== 'sqlite3') { throw new Error(`Local SQLite execution cannot run driver "${input.connection?.driver ?? 'unknown'}".`); } - if (input.connection?.readonly !== true) { - throw new Error(`Local query execution requires connections.${input.connectionId}.readonly: true.`); - } const pathValue = stringConfigValue(input.connection, 'path'); const urlValue = stringConfigValue(input.connection, 'url'); diff --git a/packages/context/src/ingest/adapters/historic-sql/redaction.test.ts b/packages/context/src/ingest/adapters/historic-sql/redaction.test.ts index c8f1d78b..d27015a6 100644 --- a/packages/context/src/ingest/adapters/historic-sql/redaction.test.ts +++ b/packages/context/src/ingest/adapters/historic-sql/redaction.test.ts @@ -9,7 +9,7 @@ describe('historic-SQL redaction', () => { ]); const sql = - "select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; + "select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; // pragma: allowlist secret expect(redactHistoricSqlText(sql, redactors)).toBe( "select * from public.api_events where api_key = '[REDACTED]' and note = '[REDACTED]'", diff --git a/packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts b/packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts index ed082181..b2af032f 100644 --- a/packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts +++ b/packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts @@ -169,7 +169,7 @@ describe('stageHistoricSqlAggregatedSnapshot', () => { it('redacts configured SQL substrings in staged artifacts while analyzing original SQL', async () => { const stagedDir = await tempDir(); const originalSql = - "select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; + "select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; // pragma: allowlist secret const reader: HistoricSqlReader = { async probe() { return { warnings: [], info: [] }; diff --git a/packages/context/src/ingest/adapters/live-database/daemon-introspection.test.ts b/packages/context/src/ingest/adapters/live-database/daemon-introspection.test.ts index fe65920e..93a9739d 100644 --- a/packages/context/src/ingest/adapters/live-database/daemon-introspection.test.ts +++ b/packages/context/src/ingest/adapters/live-database/daemon-introspection.test.ts @@ -45,7 +45,6 @@ describe('createDaemonLiveDatabaseIntrospection', () => { warehouse: { driver: 'postgres', url: 'postgres://localhost:5432/warehouse', - readonly: true, }, }, schemas: ['public'], @@ -157,7 +156,6 @@ describe('createDaemonLiveDatabaseIntrospection', () => { warehouse: { driver: 'postgresql', url: 'postgres://localhost:5432/warehouse', - readonly: true, }, }, baseUrl: `http://127.0.0.1:${address.port}`, @@ -186,20 +184,18 @@ describe('createDaemonLiveDatabaseIntrospection', () => { } }); - it('requires a configured read-only postgres connection with a url', async () => { + it('requires a configured postgres connection with a url', async () => { const introspection = createDaemonLiveDatabaseIntrospection({ connections: { warehouse: { driver: 'postgres', - url: 'postgres://localhost:5432/warehouse', - readonly: false, }, }, runJson: vi.fn(async () => daemonResponse), }); await expect(introspection.extractSchema('warehouse')).rejects.toThrow( - 'Local live-database ingest requires connections.warehouse.readonly: true.', + 'Local live-database ingest requires connections.warehouse.url.', ); }); @@ -210,7 +206,6 @@ describe('createDaemonLiveDatabaseIntrospection', () => { warehouse: { driver: 'snowflake', url: 'snowflake://example', - readonly: true, }, }, runJson, diff --git a/packages/context/src/ingest/adapters/live-database/daemon-introspection.ts b/packages/context/src/ingest/adapters/live-database/daemon-introspection.ts index 531c1a66..6c333385 100644 --- a/packages/context/src/ingest/adapters/live-database/daemon-introspection.ts +++ b/packages/context/src/ingest/adapters/live-database/daemon-introspection.ts @@ -162,9 +162,6 @@ function requirePostgresConnection( if (driver !== 'postgres') { throw new Error(`Local live-database ingest cannot run driver "${connection?.driver ?? 'unknown'}".`); } - if (connection?.readonly !== true) { - throw new Error(`Local live-database ingest requires connections.${connectionId}.readonly: true.`); - } if (typeof connection.url !== 'string' || connection.url.trim().length === 0) { throw new Error(`Local live-database ingest requires connections.${connectionId}.url.`); } diff --git a/packages/context/src/ingest/adapters/metabase/client.test.ts b/packages/context/src/ingest/adapters/metabase/client.test.ts index 1c0fdfa9..3d45a276 100644 --- a/packages/context/src/ingest/adapters/metabase/client.test.ts +++ b/packages/context/src/ingest/adapters/metabase/client.test.ts @@ -92,7 +92,7 @@ describe('MetabaseClient retry exhaustion', () => { .mockResolvedValueOnce(new Response(JSON.stringify([]), { status: 200 })); const client = new MetabaseClient( - { apiUrl: 'https://metabase.example.test', apiKey: 'key' }, + { apiUrl: 'https://metabase.example.test', apiKey: 'key' }, // pragma: allowlist secret { ...DEFAULT_METABASE_CLIENT_CONFIG, baseDelayMs: 0, diff --git a/packages/context/src/ingest/adapters/metabase/local-metabase.adapter.test.ts b/packages/context/src/ingest/adapters/metabase/local-metabase.adapter.test.ts index 7cbe913b..b25ea18b 100644 --- a/packages/context/src/ingest/adapters/metabase/local-metabase.adapter.test.ts +++ b/packages/context/src/ingest/adapters/metabase/local-metabase.adapter.test.ts @@ -39,7 +39,7 @@ describe('metabaseRuntimeConfigFromLocalConnection', () => { const connection: KtxProjectConnectionConfig = { driver: 'metabase', api_url: 'https://metabase.example.com', - api_key_ref: `file:${keyPath}`, + api_key_ref: `file:${keyPath}`, // pragma: allowlist secret }; expect(metabaseRuntimeConfigFromLocalConnection('prod-metabase', connection)).toEqual({ diff --git a/packages/context/src/ingest/local-stage-ingest.test.ts b/packages/context/src/ingest/local-stage-ingest.test.ts index 5f0ee501..3a0beaa5 100644 --- a/packages/context/src/ingest/local-stage-ingest.test.ts +++ b/packages/context/src/ingest/local-stage-ingest.test.ts @@ -39,7 +39,6 @@ async function writeLiveDatabaseConfig(projectDir: string): Promise { ' warehouse:', ' driver: postgres', ' url: postgres://localhost:5432/warehouse', - ' readonly: true', 'ingest:', ' adapters:', ' - live-database', diff --git a/packages/context/src/mcp/local-project-ports.test.ts b/packages/context/src/mcp/local-project-ports.test.ts index b95e4ad1..fab2f076 100644 --- a/packages/context/src/mcp/local-project-ports.test.ts +++ b/packages/context/src/mcp/local-project-ports.test.ts @@ -75,7 +75,6 @@ describe('createLocalProjectMcpContextPorts', () => { project.config.connections.warehouse = { driver: 'postgres', url: 'env:DATABASE_URL', - readonly: true, }; const ports = createLocalProjectMcpContextPorts(project); @@ -89,7 +88,6 @@ describe('createLocalProjectMcpContextPorts', () => { project.config.connections.warehouse = { driver: 'postgres', url: 'env:DATABASE_URL', - readonly: true, }; const connector = testConnector(); const createConnector = vi.fn(async () => connector); @@ -125,7 +123,6 @@ describe('createLocalProjectMcpContextPorts', () => { const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); project.config.connections.warehouse = { driver: 'postgres', - readonly: true, }; project.config.ingest.adapters = ['fake']; project.config.ingest.embeddings = { @@ -633,7 +630,6 @@ describe('createLocalProjectMcpContextPorts', () => { project.config.connections.warehouse = { driver: 'postgres', url: 'env:DATABASE_URL', - readonly: true, }; const shapeOnlyPorts = createLocalProjectMcpContextPorts(project); await shapeOnlyPorts.semanticLayer?.writeSource({ @@ -720,7 +716,6 @@ describe('createLocalProjectMcpContextPorts', () => { project.config.connections.warehouse = { driver: 'postgres', url: 'env:DATABASE_URL', - readonly: true, }; const shapeOnlyPorts = createLocalProjectMcpContextPorts(project); await shapeOnlyPorts.semanticLayer?.writeSource({ @@ -958,7 +953,6 @@ describe('createLocalProjectMcpContextPorts', () => { project.config.connections.warehouse = { driver: 'postgres', url: 'postgres://localhost:5432/warehouse', - readonly: true, }; project.config.ingest.adapters = ['live-database']; project.config.llm = { @@ -1034,7 +1028,6 @@ describe('createLocalProjectMcpContextPorts', () => { project.config.connections.warehouse = { driver: 'postgres', url: 'env:DATABASE_URL', - readonly: true, }; project.config.ingest.adapters = ['live-database']; const ports = createLocalProjectMcpContextPorts(project, { diff --git a/packages/context/src/memory/local-memory.test.ts b/packages/context/src/memory/local-memory.test.ts index e44a5bf1..83b22146 100644 --- a/packages/context/src/memory/local-memory.test.ts +++ b/packages/context/src/memory/local-memory.test.ts @@ -145,7 +145,7 @@ describe('createLocalProjectMemoryCapture', () => { it('captures a semantic-layer source for a named local connection id', async () => { const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); - project.config.connections.warehouse = { driver: 'postgres', readonly: true }; + project.config.connections.warehouse = { driver: 'postgres' }; const agentRunner = { runLoop: async ({ toolSet, diff --git a/packages/context/src/project/config.ts b/packages/context/src/project/config.ts index 2cf8e92b..6cd4d8fe 100644 --- a/packages/context/src/project/config.ts +++ b/packages/context/src/project/config.ts @@ -69,7 +69,6 @@ export interface KtxProjectScanConfig { export interface KtxProjectConnectionConfig { driver: string; url?: string; - readonly?: boolean; [key: string]: unknown; } diff --git a/packages/context/src/scan/local-scan.test.ts b/packages/context/src/scan/local-scan.test.ts index 2d912c54..6e9076c6 100644 --- a/packages/context/src/scan/local-scan.test.ts +++ b/packages/context/src/scan/local-scan.test.ts @@ -110,7 +110,6 @@ async function writeLiveDatabaseConfig(projectDir: string): Promise { ' warehouse:', ' driver: postgres', ' url: env:DATABASE_URL', - ' readonly: true', 'ingest:', ' adapters:', ' - live-database', @@ -1043,7 +1042,6 @@ describe('local scan', () => { ' warehouse:', ' driver: postgres', ' url: env:DATABASE_URL', - ' readonly: true', 'ingest:', ' adapters:', ' - live-database', @@ -1400,7 +1398,6 @@ describe('local scan', () => { ' warehouse:', ' driver: sqlite', ' path: warehouse.db', - ' readonly: true', 'ingest:', ' adapters:', ' - live-database', @@ -1433,7 +1430,6 @@ describe('local scan', () => { ' warehouse:', ' driver: mysql', ' url: env:MYSQL_URL', - ' readonly: true', 'ingest:', ' adapters:', ' - live-database', @@ -1469,7 +1465,6 @@ describe('local scan', () => { ' database: analytics', ' username: reader', ' password: env:CLICKHOUSE_PASSWORD', - ' readonly: true', 'ingest:', ' adapters:', ' - live-database', @@ -1505,7 +1500,6 @@ describe('local scan', () => { ' database: analytics', ' username: reader', ' schema: dbo', - ' readonly: true', 'ingest:', ' adapters:', ' - live-database', diff --git a/packages/context/src/scan/relationship-artifacts.test.ts b/packages/context/src/scan/relationship-artifacts.test.ts index e5fd0190..c366f1b2 100644 --- a/packages/context/src/scan/relationship-artifacts.test.ts +++ b/packages/context/src/scan/relationship-artifacts.test.ts @@ -24,7 +24,6 @@ async function writeWarehouseConfig(projectDir: string): Promise { ' warehouse:', ' driver: sqlite', ' path: warehouse.db', - ' readonly: true', 'ingest:', ' adapters:', ' - live-database', diff --git a/packages/context/src/scan/relationship-review-decisions.test.ts b/packages/context/src/scan/relationship-review-decisions.test.ts index 3b92e345..30277c57 100644 --- a/packages/context/src/scan/relationship-review-decisions.test.ts +++ b/packages/context/src/scan/relationship-review-decisions.test.ts @@ -28,7 +28,6 @@ async function createProject(projectDir: string): Promise { ' warehouse:', ' driver: sqlite', ' path: warehouse.db', - ' readonly: true', 'ingest:', ' adapters:', ' - live-database', diff --git a/packages/context/src/sl/local-query.test.ts b/packages/context/src/sl/local-query.test.ts index 80209329..4105c9f6 100644 --- a/packages/context/src/sl/local-query.test.ts +++ b/packages/context/src/sl/local-query.test.ts @@ -14,7 +14,7 @@ describe('compileLocalSlQuery', () => { beforeEach(async () => { tempDir = await mkdtemp(join(tmpdir(), 'ktx-local-query-')); project = await initKtxProject({ projectDir: join(tempDir, 'project'), projectName: 'warehouse' }); - project.config.connections.warehouse = { driver: 'postgres', readonly: true }; + project.config.connections.warehouse = { driver: 'postgres' }; await project.fileStore.writeFile( 'semantic-layer/warehouse/orders.yaml', `name: orders @@ -222,7 +222,7 @@ grain: [] expect(queryExecutor.execute).toHaveBeenCalledWith({ connectionId: 'warehouse', projectDir: project.projectDir, - connection: { driver: 'postgres', readonly: true }, + connection: { driver: 'postgres' }, sql: 'select status, count(*) as order_count from public.orders group by status', maxRows: 10, }); @@ -248,7 +248,7 @@ grain: [] }); it('requires connectionId when multiple connections are configured', async () => { - project.config.connections.analytics = { driver: 'bigquery', readonly: true }; + project.config.connections.analytics = { driver: 'bigquery' }; await expect( compileLocalSlQuery(project, { diff --git a/packages/llm/src/embedding-health.test.ts b/packages/llm/src/embedding-health.test.ts index ca998aa9..65956311 100644 --- a/packages/llm/src/embedding-health.test.ts +++ b/packages/llm/src/embedding-health.test.ts @@ -17,13 +17,13 @@ describe('KTX embedding health check', () => { backend: 'openai', model: 'text-embedding-3-small', dimensions: 3, - openai: { apiKey: 'sk-openai-test' }, + openai: { apiKey: 'sk-openai-test' }, // pragma: allowlist secret }, { deps: { createOpenAIClient } }, ), ).resolves.toEqual({ ok: true }); - expect(createOpenAIClient).toHaveBeenCalledWith({ apiKey: 'sk-openai-test', baseURL: undefined }); + expect(createOpenAIClient).toHaveBeenCalledWith({ apiKey: 'sk-openai-test', baseURL: undefined }); // pragma: allowlist secret }); it('returns failed when the provider returns the wrong dimensions', async () => { @@ -41,7 +41,7 @@ describe('KTX embedding health check', () => { backend: 'openai', model: 'text-embedding-3-small', dimensions: 3, - openai: { apiKey: 'sk-openai-test' }, + openai: { apiKey: 'sk-openai-test' }, // pragma: allowlist secret }, { deps: { createOpenAIClient } }, ), @@ -66,7 +66,7 @@ describe('KTX embedding health check', () => { backend: 'openai', model: 'text-embedding-3-small', dimensions: 3, - openai: { apiKey: 'sk-openai-secret' }, + openai: { apiKey: 'sk-openai-secret' }, // pragma: allowlist secret }, { deps: { createOpenAIClient } }, ), @@ -94,7 +94,7 @@ describe('KTX embedding health check', () => { backend: 'openai', model: 'text-embedding-3-small', dimensions: 3, - openai: { apiKey: 'sk-openai-test' }, + openai: { apiKey: 'sk-openai-test' }, // pragma: allowlist secret }, { timeoutMs: 1, deps: { createOpenAIClient } }, ), diff --git a/packages/llm/src/model-health.test.ts b/packages/llm/src/model-health.test.ts index d1b3df47..8752b09e 100644 --- a/packages/llm/src/model-health.test.ts +++ b/packages/llm/src/model-health.test.ts @@ -14,7 +14,7 @@ describe('KTX LLM health check', () => { runKtxLlmHealthCheck( { backend: 'anthropic', - anthropic: { apiKey: 'sk-ant-test' }, + anthropic: { apiKey: 'sk-ant-test' }, // pragma: allowlist secret modelSlots: { default: 'claude-sonnet-4-6' }, }, { deps: { createAnthropic, generateText, devtoolsEnabled: true, wrapLanguageModel } }, @@ -23,7 +23,7 @@ describe('KTX LLM health check', () => { expect(createAnthropic).toHaveBeenCalledWith( expect.objectContaining({ - apiKey: 'sk-ant-test', + apiKey: 'sk-ant-test', // pragma: allowlist secret }), ); expect(generateText).toHaveBeenCalledWith( @@ -46,7 +46,7 @@ describe('KTX LLM health check', () => { runKtxLlmHealthCheck( { backend: 'anthropic', - anthropic: { apiKey: 'sk-ant-secret' }, + anthropic: { apiKey: 'sk-ant-secret' }, // pragma: allowlist secret modelSlots: { default: 'claude-sonnet-4-6' }, }, { diff --git a/pyproject.toml b/pyproject.toml index 1c4816b7..e6422fb6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,7 @@ Issues = "https://github.com/kaelio/ktx/issues" [dependency-groups] dev = [ + "pre-commit>=4.6.0", "pytest>=9.0.2", "ruff>=0.8.4", ] diff --git a/python/ktx-daemon/src/ktx_daemon/sql_analysis.py b/python/ktx-daemon/src/ktx_daemon/sql_analysis.py index 9a222098..d5deb240 100644 --- a/python/ktx-daemon/src/ktx_daemon/sql_analysis.py +++ b/python/ktx-daemon/src/ktx_daemon/sql_analysis.py @@ -130,7 +130,9 @@ def _analyze_one( ) -def _analyze_payload(payload: tuple[str, str, str]) -> tuple[str, AnalyzeSqlBatchResult]: +def _analyze_payload( + payload: tuple[str, str, str], +) -> tuple[str, AnalyzeSqlBatchResult]: item_id, sql, dialect = payload return _analyze_one(item_id, sql, dialect) diff --git a/scripts/build-adventureworks-oltp-fixture.mjs b/scripts/build-adventureworks-oltp-fixture.mjs index 4ac2e5ba..2a57fe85 100644 --- a/scripts/build-adventureworks-oltp-fixture.mjs +++ b/scripts/build-adventureworks-oltp-fixture.mjs @@ -231,7 +231,6 @@ async function main() { driver: 'sqlserver', url, schemas: ['dbo', 'HumanResources', 'Person', 'Production', 'Purchasing', 'Sales'], - readonly: true, trustServerCertificate: true, }, now: () => new Date('2026-05-07T00:00:00.000Z'), diff --git a/scripts/examples-docs.test.mjs b/scripts/examples-docs.test.mjs index 0e985c49..b5653ee7 100644 --- a/scripts/examples-docs.test.mjs +++ b/scripts/examples-docs.test.mjs @@ -50,7 +50,6 @@ describe('standalone example docs', () => { config, /path: \.\.\/\.\.\/packages\/context\/test\/fixtures\/relationship-benchmarks\/orbit_style_product_no_declared_constraints\/data\.sqlite/, ); - assert.match(config, /readonly: true/); assert.match(config, /llm_proposals: false/); assert.match(config, /validation_required_for_manifest: true/); }); diff --git a/scripts/installed-live-database-smoke.mjs b/scripts/installed-live-database-smoke.mjs index f31d413a..fae97850 100644 --- a/scripts/installed-live-database-smoke.mjs +++ b/scripts/installed-live-database-smoke.mjs @@ -92,7 +92,6 @@ export function buildKtxYaml(postgresUrl) { ' warehouse:', ' driver: postgres', ` url: "${postgresUrl}"`, - ' readonly: true', 'storage:', ' state: sqlite', ' search: sqlite-fts5', diff --git a/scripts/installed-live-database-smoke.test.mjs b/scripts/installed-live-database-smoke.test.mjs index 3eecf053..3eda6cf0 100644 --- a/scripts/installed-live-database-smoke.test.mjs +++ b/scripts/installed-live-database-smoke.test.mjs @@ -58,7 +58,6 @@ describe('installed live-database artifact smoke helpers', () => { ' warehouse:', ' driver: postgres', ' url: "postgresql://ktx:postgres@127.0.0.1:15432/warehouse"', // pragma: allowlist secret - ' readonly: true', 'storage:', ' state: sqlite', ' search: sqlite-fts5', diff --git a/scripts/package-artifacts.mjs b/scripts/package-artifacts.mjs index 66fe1e5d..31cb8aba 100644 --- a/scripts/package-artifacts.mjs +++ b/scripts/package-artifacts.mjs @@ -617,7 +617,7 @@ try { '--skip-sources', '--skip-agents', ]); - requireProjectStderr('ktx setup', init, projectDir); + requireSuccess('ktx setup', init); requireOutput('ktx setup', init, /Project: /); const emptyProjectDir = join(root, 'empty-project'); @@ -636,7 +636,7 @@ try { '--skip-sources', '--skip-agents', ]); - requireProjectStderr('ktx setup empty project', emptyInit, emptyProjectDir); + requireSuccess('ktx setup empty project', emptyInit); await writeFile( join(projectDir, 'ktx.yaml'), [ @@ -645,7 +645,6 @@ try { ' warehouse:', ' driver: sqlite', ' path: warehouse.db', - ' readonly: true', 'storage:', ' state: sqlite', ' search: sqlite-fts5', diff --git a/scripts/public-benchmark-manifest.json b/scripts/public-benchmark-manifest.json index e106e24e..fdd97e59 100644 --- a/scripts/public-benchmark-manifest.json +++ b/scripts/public-benchmark-manifest.json @@ -4,7 +4,7 @@ "id": "chinook_with_declared_metadata", "displayName": "Chinook (SQLite, declared metadata)", "url": "https://raw.githubusercontent.com/lerocha/chinook-database/master/ChinookDatabase/DataSources/Chinook_Sqlite.sqlite", - "sha256": "7651ba378ac2fcd0dfc3c66fb101f7a7eed3ba39a612ec642b96e20702061f15", + "sha256": "7651ba378ac2fcd0dfc3c66fb101f7a7eed3ba39a612ec642b96e20702061f15", "_allowlist": "// pragma: allowlist secret", "license": "MIT", "source": "https://github.com/lerocha/chinook-database" }, @@ -12,7 +12,7 @@ "id": "northwind_with_declared_metadata", "displayName": "Northwind (SQLite, declared metadata)", "url": "https://github.com/jpwhite3/northwind-SQLite3/raw/main/dist/northwind.db", - "sha256": "2f4f5c68dfcd33ba27373eae48c7a4869800c68095ee0f9f0da494f83382a877", + "sha256": "2f4f5c68dfcd33ba27373eae48c7a4869800c68095ee0f9f0da494f83382a877", "_allowlist": "// pragma: allowlist secret", "license": "MIT", "source": "https://github.com/jpwhite3/northwind-SQLite3" }, @@ -20,7 +20,7 @@ "id": "sakila_with_declared_metadata", "displayName": "Sakila (SQLite, declared metadata)", "url": "https://raw.githubusercontent.com/bradleygrant/sakila-sqlite3/master/sakila_master.db", - "sha256": "88c91a4a1a6b61f9d3f35904c0a173c887b25e73f20c3c2fdb073818c06f4268", + "sha256": "88c91a4a1a6b61f9d3f35904c0a173c887b25e73f20c3c2fdb073818c06f4268", "_allowlist": "// pragma: allowlist secret", "license": "BSD-2-Clause", "source": "https://github.com/bradleygrant/sakila-sqlite3" }, diff --git a/scripts/standalone-ci-workflow.test.mjs b/scripts/standalone-ci-workflow.test.mjs index 195fce53..5aa4cc02 100644 --- a/scripts/standalone-ci-workflow.test.mjs +++ b/scripts/standalone-ci-workflow.test.mjs @@ -20,6 +20,8 @@ describe('standalone KTX CI workflow', () => { assertIncludesAll(workflow, [ 'permissions:', 'contents: read', + 'pre-commit-checks:', + 'name: Pre-commit checks', 'typescript-checks:', 'name: TypeScript checks', 'slow-context-tests:', @@ -33,7 +35,7 @@ describe('standalone KTX CI workflow', () => { 'artifact-checks:', 'name: Artifact checks', 'actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd', - 'pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0', + 'pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093', 'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e', 'node-version: "24"', 'cache-dependency-path: "pnpm-lock.yaml"', @@ -46,7 +48,10 @@ describe('standalone KTX CI workflow', () => { 'actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405', 'python-version: "3.13"', 'astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b', + 'version: "0.11.11"', 'cache-dependency-glob: "uv.lock"', + 'uv sync --all-packages --all-groups', + 'uv run pre-commit run --all-files', 'uv sync --all-packages', 'uv run pytest', 'pnpm run artifacts:check', diff --git a/uv.lock b/uv.lock index 5458900e..5531c8e3 100644 --- a/uv.lock +++ b/uv.lock @@ -546,6 +546,7 @@ source = { virtual = "." } [package.dev-dependencies] dev = [ + { name = "pre-commit" }, { name = "pytest" }, { name = "ruff" }, ] @@ -554,6 +555,7 @@ dev = [ [package.metadata.requires-dev] dev = [ + { name = "pre-commit", specifier = ">=4.6.0" }, { name = "pytest", specifier = ">=9.0.2" }, { name = "ruff", specifier = ">=0.8.4" }, ]