apunkt/ktx
1
0
Fork 0
mirror of https://github.com/Kaelio/ktx.git synced 2026-06-10 08:05:14 +02:00
Code Issues Projects Releases Packages Wiki Activity

Merge origin/main into merge-scan-into-ingest-v1

Browse source
This commit is contained in:
Andrey Avtomonov 2026-05-14 00:47:03 +02:00
commit 9131c82724
98 changed files with 3207 additions and 1007 deletions
Download patch file Download diff file Expand all files Collapse all files

50
.github/workflows/ci.yml vendored
View file

@ -15,6 +15,46 @@ concurrency:
cancel-in-progress: true
jobs:
pre-commit-checks:
name: Pre-commit checks
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Setup pnpm
uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8
with:
run_install: false
- name: Setup Node.js
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
with:
node-version: "24"
cache: "pnpm"
cache-dependency-path: "pnpm-lock.yaml"
- name: Install TypeScript dependencies
run: pnpm install --frozen-lockfile
- name: Setup Python
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
with:
python-version: "3.13"
- name: Setup uv
uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
with:
version: "0.11.11"
enable-cache: true
cache-dependency-glob: "uv.lock"
- name: Install Python dependencies
run: uv sync --all-packages --all-groups
- name: Run pre-commit hooks
run: uv run pre-commit run --all-files
typescript-checks:
name: TypeScript checks
runs-on: ubuntu-latest
@ -23,7 +63,7 @@ jobs:
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Setup pnpm
uses: pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0 # v6.0.7
uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8
with:
run_install: false
@ -51,7 +91,7 @@ jobs:
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Setup pnpm
uses: pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0 # v6.0.7
uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8
with:
run_install: false
@ -79,7 +119,7 @@ jobs:
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Setup pnpm
uses: pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0 # v6.0.7
uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8
with:
run_install: false
@ -107,7 +147,7 @@ jobs:
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Setup pnpm
uses: pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0 # v6.0.7
uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8
with:
run_install: false
@ -156,7 +196,7 @@ jobs:
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Setup pnpm
uses: pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0 # v6.0.7
uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8
with:
run_install: false

2
.github/workflows/release.yml vendored
View file

@ -24,7 +24,7 @@ jobs:
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Setup pnpm
uses: pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0 # v6.0.7
uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8
with:
run_install: false

1
LICENSE
View file

@ -199,4 +199,3 @@
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

3
docs-site/content/docs/cli-reference/ktx-connection.mdx
View file

@ -63,8 +63,7 @@ agents.
"connections": [
{
"id": "my-warehouse",
"driver": "postgres",
"readonly": false
"driver": "postgres"
}
]
}

93
docs-site/content/docs/cli-reference/ktx-setup.mdx
View file

@ -18,8 +18,6 @@ ktx setup [options]
| Flag | Description | Default |
|------|-------------|---------|
| `--project-dir <path>` | KTX project directory | `KTX_PROJECT_DIR`, nearest `ktx.yaml`, or cwd |
| `--new` | Create a new KTX project before setup | `false` |
| `--existing` | Use an existing KTX project | `false` |
| `--yes` | Accept safe defaults in non-interactive setup | `false` |
| `--no-input` | Disable interactive terminal input | — |
@ -29,75 +27,11 @@ ktx setup [options]
|------|-------------|---------|
| `--agents` | Install agent integration only | `false` |
| `--target <target>` | Agent target (`claude-code`, `codex`, `cursor`, `opencode`, `universal`) | — |
| `--agent-scope <scope>` | Agent install scope (`project` or `global`) | `project` |
| `--project` | Install agent integration into the project scope | `false` |
| `--global` | Install agent integration into the global target scope (Claude Code and Codex only) | `false` |
| `--skip-agents` | Leave agent integration incomplete for now | `false` |
### LLM Configuration
| Flag | Description | Default |
|------|-------------|---------|
| `--anthropic-api-key-env <name>` | Environment variable containing the Anthropic API key | — |
| `--anthropic-api-key-file <path>` | File containing the Anthropic API key | — |
| `--anthropic-model <model>` | Anthropic model ID to validate and save | — |
| `--skip-llm` | Leave LLM setup incomplete for now | `false` |
### Embedding Configuration
| Flag | Description | Default |
|------|-------------|---------|
| `--embedding-backend <backend>` | Embedding backend (`openai` or `sentence-transformers`) | — |
| `--embedding-api-key-env <name>` | Environment variable containing the embedding provider API key | — |
| `--embedding-api-key-file <path>` | File containing the embedding provider API key | — |
| `--skip-embeddings` | Leave embedding setup incomplete for now | `false` |
### Database Configuration
| Flag | Description | Default |
|------|-------------|---------|
| `--database <driver>` | Database driver to configure; repeatable (`sqlite`, `postgres`, `mysql`, `clickhouse`, `sqlserver`, `bigquery`, `snowflake`) | — |
| `--database-connection-id <id>` | Existing or new connection id; repeatable | — |
| `--new-database-connection-id <id>` | Connection id for one new database connection | — |
| `--database-url <url>` | URL, `env:NAME`, or `file:/path` for one new URL-style database connection | — |
| `--database-schema <schema>` | Database schema to include; repeatable | — |
| `--skip-databases` | Leave database setup incomplete | `false` |
### Query history
| Flag | Description | Default |
|------|-------------|---------|
| `--enable-query-history` | Enable query history when the selected database supports it | `false` |
| `--disable-query-history` | Disable query history for the selected database | `false` |
| `--query-history-window-days <number>` | Query-history lookback window in days | — |
| `--query-history-min-executions <number>` | Minimum executions for a query-history template | — |
| `--query-history-service-account-pattern <pattern>` | Query-history service-account regex; repeatable | — |
| `--query-history-redaction-pattern <pattern>` | Query-history SQL-literal redaction regex; repeatable | — |
### Context Source Configuration
| Flag | Description | Default |
|------|-------------|---------|
| `--source <type>` | Source connector type (`dbt`, `metricflow`, `metabase`, `looker`, `lookml`, `notion`) | — |
| `--source-connection-id <id>` | Connection id for source setup | — |
| `--source-path <path>` | Local source path for dbt, MetricFlow, or LookML | — |
| `--source-git-url <url>` | Git URL for dbt, MetricFlow, or LookML | — |
| `--source-branch <branch>` | Git branch for source setup | — |
| `--source-subpath <path>` | Repo subpath for source setup | — |
| `--source-auth-token-ref <ref>` | `env:` or `file:` credential ref for source repo auth | — |
| `--source-url <url>` | Source service URL for Metabase or Looker | — |
| `--source-api-key-ref <ref>` | `env:` or `file:` API key ref for Metabase or Notion | — |
| `--source-client-id <id>` | Looker client id | — |
| `--source-client-secret-ref <ref>` | `env:` or `file:` Looker client secret ref | — |
| `--source-warehouse-connection-id <id>` | Mapped warehouse connection id | — |
| `--source-project-name <name>` | dbt project name override | — |
| `--source-profiles-path <path>` | dbt profiles path | — |
| `--source-target <target>` | dbt target or source-specific mapping target | — |
| `--metabase-database-id <id>` | Metabase database id to map | — |
| `--notion-crawl-mode <mode>` | Notion crawl mode (`all_accessible` or `selected_roots`) | — |
| `--notion-root-page-id <id>` | Notion root page id; repeatable | — |
| `--skip-initial-source-ingest` | Validate source setup without building source context during setup | `false` |
| `--skip-sources` | Mark optional source setup complete with no sources | `false` |
The setup wizard is the public configuration interface. It prompts for LLM
credentials, embeddings, database connections, context sources, query history,
and agent integration when those values are needed.
## Examples
@ -105,17 +39,8 @@ ktx setup [options]
# Run the interactive setup wizard
ktx setup
# Create a new project and run setup
ktx setup --new
# Resume setup in an existing project
ktx setup --existing
# Non-interactive setup with Anthropic key from environment
ktx setup --yes --anthropic-api-key-env ANTHROPIC_API_KEY
# Set up a Postgres connection
ktx setup --database postgres --database-url "env:DATABASE_URL"
# Run setup for a specific project directory
ktx setup --project-dir ./analytics
# Install agent integration for Claude Code only
ktx setup --agents --target claude-code
@ -123,12 +48,6 @@ ktx setup --agents --target claude-code
# Install agent integration globally for Codex
ktx setup --agents --target codex --global
# Add a dbt source from a local path
ktx setup --source dbt --source-path ./my-dbt-project
# Skip optional steps for a minimal setup
ktx setup --skip-sources --skip-agents
# Check setup readiness
ktx status
```
@ -155,5 +74,5 @@ Agent integration ready: yes (codex:project)
|-------|-------|----------|
| Setup resumes an unexpected project | `KTX_PROJECT_DIR` or nearest `ktx.yaml` points to another directory | Pass `--project-dir <path>` explicitly |
| Health check for model fails | Provider key or model id is invalid | Set the correct environment variable or secret file and rerun setup |
| Setup cannot run in CI | Interactive prompts need a TTY | Use `--yes --no-input` with explicit flags for required values |
| Setup cannot run in CI | Interactive prompts need a TTY | Run setup interactively before CI, or provide a fixture `ktx.yaml` for automated tests |
| Agent integration missing | Setup skipped the agents step | Run `ktx setup --agents --target <target>` |

2
docs-site/content/docs/getting-started/quickstart.mdx
View file

@ -244,7 +244,7 @@ Agent integration ready: yes (claude-code:project)
| Local embeddings hang or fail | The managed Python runtime cannot start or the local model runtime is unavailable | Install `uv`, run `ktx dev runtime status`, then run `ktx dev runtime install --feature local-embeddings --yes` and rerun setup |
| Database connection test fails | Credentials, network access, warehouse, database, or schema value is wrong | Test the same URL with the database's native client, then rerun `ktx setup` and reconfigure the connection |
| `KTX context built: no` in `ktx status` | Setup saved configuration but did not build context | Run `ktx setup` and choose to build context now |
| Agent integration is incomplete | Setup skipped the agents step or the target was not installed | Run `ktx setup --agents --target codex --project` using the target you need |
| Agent integration is incomplete | Setup skipped the agents step or the target was not installed | Run `ktx setup --agents --target codex` using the target you need |
## Next steps

9
docs-site/content/docs/integrations/context-sources.mdx
View file

@ -24,7 +24,6 @@ Agents must configure and ingest context sources in this order:
| Field | Required | Description |
|-------|----------|-------------|
| `driver` | Yes | Source adapter: `dbt`, `metricflow`, `lookml`, `metabase`, `looker`, or `notion` |
| `readonly` | Strongly recommended | Marks the source as read-only for KTX |
| `source_dir` | For local file sources | Absolute or project-relative source directory |
| `repo_url` | For Git-hosted sources | Git repository URL |
| `branch` | No | Git branch to read |
@ -50,7 +49,6 @@ connections:
my-dbt:
driver: dbt
source_dir: /path/to/dbt/project
readonly: true
```
For a Git-hosted project:
@ -63,7 +61,6 @@ connections:
branch: main
path: analytics/dbt # For monorepos
auth_token_ref: env:GITHUB_TOKEN
readonly: true
```
### Authentication
@ -111,7 +108,6 @@ connections:
branch: main
path: dbt_metrics # Subdirectory for monorepos
auth_token_ref: env:GITHUB_TOKEN
readonly: true
```
For a local path:
@ -158,7 +154,6 @@ connections:
branch: main
path: analytics # Subdirectory for monorepos
auth_token_ref: env:GITHUB_TOKEN
readonly: true
```
For a local path:
@ -220,7 +215,6 @@ connections:
syncEnabled:
"3": true
syncMode: ONLY # Only ingest mapped databases
readonly: true
```
### Authentication
@ -277,7 +271,6 @@ connections:
mappings:
connectionMappings:
postgres_connection: postgres-main # Looker conn → KTX conn
readonly: true
```
### Authentication
@ -330,7 +323,6 @@ connections:
crawl_mode: selected_roots
root_page_ids:
- "abc123def456..."
readonly: true
```
For crawling all accessible pages:
@ -341,7 +333,6 @@ connections:
driver: notion
auth_token_ref: env:NOTION_TOKEN
crawl_mode: all_accessible
readonly: true
```
### Authentication

18
docs-site/content/docs/integrations/primary-sources.mdx
View file

@ -25,7 +25,6 @@ Agents should prefer environment or file references over literal secrets.
| `url` | One of the connection methods | URL-style connectors | Database URL, `env:NAME`, or `file:/path/to/secret` |
| `host`, `port`, `database`, `username`, `password` | One of the connection methods | PostgreSQL, MySQL, ClickHouse, SQL Server | Field-by-field connection values |
| `schema` or `schemas` | No | schema-aware warehouses | Single schema or list of schemas to scan |
| `readonly` | Strongly recommended | all primary sources | Marks the connection as read-only in KTX config |
| `context.queryHistory` | No | PostgreSQL, Snowflake, BigQuery | Enables query-history ingestion when the warehouse supports it |
| `path` | Yes for path-style SQLite | SQLite | Local SQLite database path or `env:NAME` reference |
@ -39,9 +38,8 @@ The most full-featured connector. Supports schema introspection, foreign key det
connections:
my-postgres:
driver: postgres
url: postgresql://user:password@host:5432/database
url: env:DATABASE_URL
schema: public
readonly: true
```
Or with individual fields:
@ -59,7 +57,6 @@ connections:
- public
- analytics
ssl: true
readonly: true
```
### Authentication
@ -128,7 +125,6 @@ connections:
username: KTX_SERVICE
password: env:SNOWFLAKE_PASSWORD
role: ANALYST
readonly: true
```
For multiple schemas:
@ -201,7 +197,6 @@ connections:
credentials_json: file:~/.config/gcloud/bq-service-account.json
dataset_id: analytics
location: US
readonly: true
```
For multiple datasets:
@ -274,7 +269,6 @@ connections:
my-clickhouse:
driver: clickhouse
url: http://localhost:8123/analytics
readonly: true
```
Or with individual fields:
@ -289,7 +283,6 @@ connections:
username: default
password: env:CH_PASSWORD
ssl: false
readonly: true
```
### Authentication
@ -332,8 +325,7 @@ Standard MySQL/MariaDB connector with full foreign key support and schema intros
connections:
my-mysql:
driver: mysql
url: mysql://user:password@host:3306/database
readonly: true
url: env:MYSQL_DATABASE_URL
```
Or with individual fields:
@ -348,7 +340,6 @@ connections:
username: ktx_reader
password: env:MYSQL_PASSWORD
ssl: true
readonly: true
```
### Authentication
@ -391,8 +382,7 @@ Connects to Microsoft SQL Server and Azure SQL. Supports multi-schema scanning w
connections:
my-sqlserver:
driver: sqlserver
url: mssql://user:password@host:1433/database?trustServerCertificate=true
readonly: true
url: env:SQLSERVER_DATABASE_URL
```
Or with individual fields:
@ -408,7 +398,6 @@ connections:
password: env:MSSQL_PASSWORD
schema: dbo
trustServerCertificate: true
readonly: true
```
For multiple schemas:
@ -460,7 +449,6 @@ connections:
my-sqlite:
driver: sqlite
path: ./data/warehouse.sqlite
readonly: true
```
Path supports multiple formats:

32
docs/superpowers/plans/2026-05-11-historic-sql-cross-dialect-readiness.md
View file

@ -40,37 +40,37 @@ This plan does not update `examples/postgres-historic/README.md` or `examples/po
Modify:
- `packages/context/src/ingest/adapters/historic-sql/types.ts`
- `packages/context/src/ingest/adapters/historic-sql/types.ts`
Adds optional probe `info` notes and lets injected historic-SQL dependencies use any reader/query client pair while preserving the existing Postgres-specific option.
- `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.ts`
- `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.ts`
Moves low `pg_stat_statements.max` from `warnings` to `info`.
- `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.test.ts`
- `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.test.ts`
Locks `track = none` as warning and low `max` as info.
- `packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.ts`
- `packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.ts`
Returns `{ warnings: [], info: [] }` from `probe()`.
- `packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.test.ts`
- `packages/context/src/ingest/adapters/historic-sql/bigquery-query-history-reader.test.ts`
Locks the BigQuery probe return object.
- `packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.ts`
- `packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.ts`
Returns `{ warnings: [], info: [] }` from `probe()`.
- `packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.test.ts`
- `packages/context/src/ingest/adapters/historic-sql/snowflake-query-history-reader.test.ts`
Locks the Snowflake probe return object.
- `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts`
- `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts`
Updates test readers to return the normalized probe shape.
- `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts`
- `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts`
Updates test readers to return the normalized probe shape.
- `packages/context/src/ingest/local-adapters.ts`
- `packages/context/src/ingest/local-adapters.ts`
Accepts generic historic-SQL reader/query-client dependencies while keeping `postgresQueryClient` as the compatibility input used by current callers.
- `packages/context/src/ingest/local-adapters.test.ts`
- `packages/context/src/ingest/local-adapters.test.ts`
Verifies generic reader/query-client injection and the existing Postgres compatibility path.
- `packages/cli/src/local-adapters.ts`
- `packages/cli/src/local-adapters.ts`
Chooses Postgres, BigQuery, or Snowflake historic-SQL readers/query clients from the configured connection.
- `packages/cli/src/local-adapters.test.ts`
- `packages/cli/src/local-adapters.test.ts`
Adds direct tests for CLI local adapter registration for Postgres, BigQuery, and Snowflake.
- `packages/cli/src/historic-sql-doctor.ts`
- `packages/cli/src/historic-sql-doctor.ts`
Treats info-only Postgres probe notes as a passing doctor check, and warnings as warnings.
- `packages/cli/src/historic-sql-doctor.test.ts`
- `packages/cli/src/historic-sql-doctor.test.ts`
Verifies low `pg_stat_statements.max` is pass/detail, while `track = none` remains warn.
- `packages/cli/src/doctor.test.ts`
- `packages/cli/src/doctor.test.ts`
Updates the project doctor integration expectation for the new info-only behavior.
## Task 1: Normalize Historic-SQL Probe Results

2
docs/superpowers/plans/2026-05-11-historic-sql-end-to-end-retrieval-acceptance.md
View file

@ -44,7 +44,7 @@ Remaining acceptance gap this plan covers:
Create:
- `packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts`
- `packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts`
Owns the end-to-end local regression for the redesigned historic-SQL pipeline. It uses the real adapter and local ingest runner, with fake deterministic reader/analysis/agent components so the test does not need a live database or LLM provider.
## Task 1: Add Real-Adapter Local Ingest Acceptance Coverage

42
docs/superpowers/plans/2026-05-11-historic-sql-foundations.md
View file

@ -41,50 +41,50 @@ The next plan after this one should cover search enrichment from spec §6.2.3-§
Create:
- `packages/context/src/ingest/adapters/historic-sql/skill-schemas.ts`
- `packages/context/src/ingest/adapters/historic-sql/skill-schemas.ts`
Owns the shared zod schemas for historic-SQL LLM outputs.
- `packages/context/src/ingest/adapters/historic-sql/skill-schemas.test.ts`
- `packages/context/src/ingest/adapters/historic-sql/skill-schemas.test.ts`
Locks schema acceptance, JSON schema generation, and future-key tolerance.
- `python/ktx-daemon/src/ktx_daemon/sql_analysis.py`
- `python/ktx-daemon/src/ktx_daemon/sql_analysis.py`
Implements batch sqlglot parsing for table and clause-level column extraction.
- `python/ktx-daemon/tests/test_sql_analysis.py`
- `python/ktx-daemon/tests/test_sql_analysis.py`
Tests batch parser behavior without FastAPI.
Modify:
- `packages/context/src/ingest/index.ts`
- `packages/context/src/ingest/index.ts`
Exports the new historic-SQL skill schemas.
- `packages/context/src/sl/types.ts`
- `packages/context/src/sl/types.ts`
Adds `usage?: TableUsageOutput` to `SemanticLayerSource`.
- `packages/context/src/sl/schemas.ts`
- `packages/context/src/sl/schemas.ts`
Accepts `usage` in standalone and overlay semantic-layer source validation.
- `packages/context/src/sl/semantic-layer.service.ts`
- `packages/context/src/sl/semantic-layer.service.ts`
Projects manifest `usage` onto `SemanticLayerSource` and composes overlay usage intentionally.
- `packages/context/src/sl/semantic-layer.service.test.ts`
- `packages/context/src/sl/semantic-layer.service.test.ts`
Tests source schema acceptance, manifest projection, and overlay composition.
- `packages/context/src/ingest/adapters/live-database/manifest.ts`
- `packages/context/src/ingest/adapters/live-database/manifest.ts`
Adds `LiveDatabaseManifestTableEntry.usage`, existing-usage inputs, and `mergeUsagePreservingExternal()`.
- `packages/context/src/ingest/adapters/live-database/manifest.test.ts`
- `packages/context/src/ingest/adapters/live-database/manifest.test.ts`
Tests scan-managed usage replacement while preserving external keys.
- `packages/context/src/scan/local-enrichment-artifacts.ts`
- `packages/context/src/scan/local-enrichment-artifacts.ts`
Loads existing manifest usage and passes it through scan manifest rebuilds.
- `packages/context/src/scan/local-enrichment-artifacts.test.ts`
- `packages/context/src/scan/local-enrichment-artifacts.test.ts`
Tests that structural scan rewrites preserve existing usage.
- `python/ktx-daemon/src/ktx_daemon/app.py`
- `python/ktx-daemon/src/ktx_daemon/app.py`
Registers `/sql/analyze-batch`.
- `python/ktx-daemon/tests/test_app.py`
- `python/ktx-daemon/tests/test_app.py`
Tests the FastAPI endpoint.
- `packages/context/src/sql-analysis/ports.ts`
- `packages/context/src/sql-analysis/ports.ts`
Adds batch analysis types and `SqlAnalysisPort.analyzeBatch()`.
- `packages/context/src/sql-analysis/index.ts`
- `packages/context/src/sql-analysis/index.ts`
Exports the new batch analysis types.
- `packages/context/src/sql-analysis/http-sql-analysis-port.ts`
- `packages/context/src/sql-analysis/http-sql-analysis-port.ts`
Maps `/sql/analyze-batch` request and response payloads.
- `packages/context/src/sql-analysis/http-sql-analysis-port.test.ts`
- `packages/context/src/sql-analysis/http-sql-analysis-port.test.ts`
Tests HTTP mapping and malformed response rejection.
- `packages/cli/src/managed-python-http.test.ts`
- `packages/cli/src/managed-python-http.test.ts`
Verifies the managed daemon wrapper routes `analyzeBatch()`.
- Existing test files with `SqlAnalysisPort` object literals
- Existing test files with `SqlAnalysisPort` object literals
Add a no-op `analyzeBatch: async () => new Map()` while legacy paths still use `analyzeForFingerprint()`.
## Task 1: Add Historic SQL Skill Schemas

8
docs/superpowers/plans/2026-05-11-historic-sql-pattern-shard-smoke-docs.md
View file

@ -39,13 +39,13 @@ Remaining gap this plan fixes:
## File Structure
- Modify `scripts/examples-docs.test.mjs`
- Modify `scripts/examples-docs.test.mjs`
Pins docs and smoke script to the sharded pattern WorkUnit contract.
- Modify `examples/postgres-historic/scripts/smoke.sh`
- Modify `examples/postgres-historic/scripts/smoke.sh`
Validates `patterns-input/part-*.json` shard files and `historic-sql-patterns-part-*` stage-only WorkUnits.
- Modify `examples/postgres-historic/README.md`
- Modify `examples/postgres-historic/README.md`
Documents `patterns-input.json` as the full audit artifact and `patterns-input/part-*.json` as bounded pattern WorkUnit input.
- Modify `examples/README.md`
- Modify `examples/README.md`
Updates the short example catalog entry with the same audit-vs-shard wording.
### Task 1: Pin Example Tests To Pattern Shards

18
docs/superpowers/plans/2026-05-11-historic-sql-pattern-workunit-sharding.md
View file

@ -30,23 +30,23 @@ No existing spec-derived plan is currently unimplemented in this worktree. This
## File Structure
- Create `packages/context/src/ingest/adapters/historic-sql/pattern-inputs.ts`
- Create `packages/context/src/ingest/adapters/historic-sql/pattern-inputs.ts`
Owns deterministic pattern audit ordering, cross-table candidate filtering, byte-bounded shard creation, shard path constants, and shard path detection.
- Create `packages/context/src/ingest/adapters/historic-sql/pattern-inputs.test.ts`
- Create `packages/context/src/ingest/adapters/historic-sql/pattern-inputs.test.ts`
Covers deterministic shard ordering, single-table exclusion from WorkUnit shards, byte limits, and oversize-template manifest warnings.
- Modify `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts`
- Modify `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts`
Writes full `patterns-input.json` plus bounded `patterns-input/part-0001.json` shard files, and appends shard warnings to `manifest.json`.
- Modify `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts`
- Modify `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts`
Adds a regression for audit file preservation and sharded WorkUnit input creation.
- Modify `packages/context/src/ingest/adapters/historic-sql/chunk-unified.ts`
- Modify `packages/context/src/ingest/adapters/historic-sql/chunk-unified.ts`
Emits one patterns WorkUnit per changed shard path, treats root `patterns-input.json` as audit-only, and includes shard paths in the scope descriptor and eviction calculation.
- Modify `packages/context/src/ingest/adapters/historic-sql/chunk-unified.test.ts`
- Modify `packages/context/src/ingest/adapters/historic-sql/chunk-unified.test.ts`
Updates root-file expectations and adds multi-shard diff behavior.
- Modify `packages/context/skills/historic_sql_patterns/SKILL.md`
- Modify `packages/context/skills/historic_sql_patterns/SKILL.md`
Tells the skill to read the exact pattern shard in `rawFiles` and emit evidence with that shard as `rawPath`.
- Modify `packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts`
- Modify `packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts`
Updates the fake agent to emit pattern evidence for `historic-sql-patterns-part-0001`.
- Modify `packages/context/src/ingest/ingest-runtime-assets.test.ts`
- Modify `packages/context/src/ingest/ingest-runtime-assets.test.ts`
Keeps packaged skill assertions aligned with sharded pattern file guidance.
## Task 1: Add Pattern Input Sharding Helper

12
docs/superpowers/plans/2026-05-11-historic-sql-redaction-hardening.md
View file

@ -55,16 +55,16 @@ Remaining spec gap this plan covers:
Create:
- `packages/context/src/ingest/adapters/historic-sql/redaction.ts`
- `packages/context/src/ingest/adapters/historic-sql/redaction.ts`
Owns compilation and application of historic-SQL SQL-text redaction patterns. Supports JavaScript regex strings and the documented `(?i)` case-insensitive prefix used by setup tests/docs.
- `packages/context/src/ingest/adapters/historic-sql/redaction.test.ts`
- `packages/context/src/ingest/adapters/historic-sql/redaction.test.ts`
Tests raw regex replacement, `(?i)` compatibility, empty config behavior, and invalid-pattern diagnostics.
Modify:
- `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts`
- `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts`
Compiles `config.redactionPatterns` once per fetch. Keeps original SQL for filtering and `SqlAnalysisPort.analyzeBatch()`, then stores redacted SQL in `ParsedTemplate.template.canonicalSql` before `toStagedTable()` and `toPatternsInput()` serialize files.
- `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts`
- `packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts`
Adds a regression proving raw secrets are absent from staged artifacts while `analyzeBatch()` still receives the original SQL.
## Task 1: Add Historic SQL Redaction Helper
@ -89,7 +89,7 @@ describe('historic-SQL redaction', () => {
]);
const sql =
"select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'";
"select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; // pragma: allowlist secret
expect(redactHistoricSqlText(sql, redactors)).toBe(
"select * from public.api_events where api_key = '[REDACTED]' and note = '[REDACTED]'",
@ -202,7 +202,7 @@ Append this test inside the existing `describe('stageHistoricSqlAggregatedSnapsh
it('redacts configured SQL substrings in staged artifacts while analyzing original SQL', async () => {
const stagedDir = await tempDir();
const originalSql =
"select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'";
"select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; // pragma: allowlist secret
const reader: HistoricSqlReader = {
async probe() {
return { warnings: [], info: [] };

22
docs/superpowers/plans/2026-05-11-historic-sql-search-enrichment.md
View file

@ -37,27 +37,27 @@ This plan does not rewrite the historic-SQL adapter, readers, skills, projection
Modify:
- `packages/context/src/sl/sl-search.service.ts`
- `packages/context/src/sl/sl-search.service.ts`
Adds usage narrative, frequency, filters, group-bys, joins, and stale marker to the canonical SL search text. Preserves snippets returned by repository search for direct `SlSearchService.search()` callers.
- `packages/context/src/sl/sl-search.service.test.ts`
- `packages/context/src/sl/sl-search.service.test.ts`
Tests usage search-text content and direct service snippet pass-through.
- `packages/context/src/sl/ports.ts`
- `packages/context/src/sl/ports.ts`
Extends `SlSourcesIndexPort.search()` rows with optional `snippet`.
- `packages/context/src/sl/sqlite-sl-sources-index.ts`
- `packages/context/src/sl/sqlite-sl-sources-index.ts`
Adds FTS5 `snippet()` selection to lexical candidate search and direct index search.
- `packages/context/src/sl/sqlite-sl-sources-index.test.ts`
- `packages/context/src/sl/sqlite-sl-sources-index.test.ts`
Locks snippet behavior for both direct search and lexical lane candidates.
- `packages/context/src/sl/local-sl.ts`
- `packages/context/src/sl/local-sl.ts`
Adds `frequencyTier` and `snippet` to query-mode `LocalSlSourceSearchResult`; collects snippets from the lexical lane and hydrates frequency from `SemanticLayerSource.usage`.
- `packages/context/src/sl/local-sl.test.ts`
- `packages/context/src/sl/local-sl.test.ts`
Tests that usage-only terms can find a source and that results include `frequencyTier` and FTS snippet.
- `packages/context/src/sl/pglite-sl-search-prototype.ts`
- `packages/context/src/sl/pglite-sl-search-prototype.ts`
Propagates `frequencyTier` for the prototype backend so the shared result type stays truthful.
- `packages/context/src/mcp/types.ts`
- `packages/context/src/mcp/types.ts`
Adds `frequencyTier` and `snippet` to `KtxSemanticLayerSourceSummary`.
- `packages/context/src/mcp/local-project-ports.ts`
- `packages/context/src/mcp/local-project-ports.ts`
Includes `frequencyTier` and `snippet` in `semanticLayer.listSources()` output.
- `packages/context/src/mcp/local-project-ports.test.ts`
- `packages/context/src/mcp/local-project-ports.test.ts`
Tests the agent/MCP-facing list response.
## Task 1: Index Historic SQL Usage In SL Search Text

50
docs/superpowers/plans/2026-05-11-historic-sql-skills-projection-cutover.md
View file

@ -52,58 +52,58 @@ Still not implemented:
Create:
- `packages/context/src/ingest/adapters/historic-sql/evidence.ts`
- `packages/context/src/ingest/adapters/historic-sql/evidence.ts`
Owns typed evidence envelopes, ignored evidence path helpers, and load/write helpers for table usage and pattern evidence.
- `packages/context/src/ingest/adapters/historic-sql/evidence.test.ts`
- `packages/context/src/ingest/adapters/historic-sql/evidence.test.ts`
Tests evidence schema validation, path normalization, and loader rejection of malformed evidence.
- `packages/context/src/ingest/adapters/historic-sql/evidence-tool.ts`
- `packages/context/src/ingest/adapters/historic-sql/evidence-tool.ts`
Adds `emit_historic_sql_evidence`, the only write tool the two new historic-SQL skills use.
- `packages/context/src/ingest/adapters/historic-sql/evidence-tool.test.ts`
- `packages/context/src/ingest/adapters/historic-sql/evidence-tool.test.ts`
Tests the tool writes ignored run-local JSON with `skipLock: true` and rejects non-historic ingest sessions.
- `packages/context/src/ingest/adapters/historic-sql/projection.ts`
- `packages/context/src/ingest/adapters/historic-sql/projection.ts`
Projects table usage evidence into manifest shards, writes pattern wiki pages, marks stale usage/pages, and deletes legacy query pages.
- `packages/context/src/ingest/adapters/historic-sql/projection.test.ts`
- `packages/context/src/ingest/adapters/historic-sql/projection.test.ts`
Tests `_schema` merge, stale usage, pattern slug reuse, stale page tagging, archive movement, and legacy page cleanup.
- `packages/context/src/ingest/adapters/historic-sql/post-processor.ts`
- `packages/context/src/ingest/adapters/historic-sql/post-processor.ts`
Implements `IngestBundlePostProcessorPort` for the deterministic projection phase.
- `packages/context/src/ingest/adapters/historic-sql/post-processor.test.ts`
- `packages/context/src/ingest/adapters/historic-sql/post-processor.test.ts`
Tests post-processor path resolution from `workdir`, `connectionId`, `sourceKey`, and `syncId`.
- `packages/context/skills/historic_sql_table_digest/SKILL.md`
- `packages/context/skills/historic_sql_table_digest/SKILL.md`
Skill for one changed `tables/*.json` WorkUnit; emits one table usage evidence object.
- `packages/context/skills/historic_sql_patterns/SKILL.md`
- `packages/context/skills/historic_sql_patterns/SKILL.md`
Skill for `patterns-input.json`; emits one pattern evidence object per recurring cross-table intent.
Modify:
- `packages/context/src/ingest/adapters/historic-sql/types.ts`
- `packages/context/src/ingest/adapters/historic-sql/types.ts`
Keep only unified config/staged schemas and reader contracts; extend config preprocessing for existing `serviceAccountUserPatterns` and `minCalls` aliases.
- `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts`
- `packages/context/src/ingest/adapters/historic-sql/stage-unified.ts`
Add `staleArchiveAfterDays` to `manifest.json` so projection can archive stale pattern pages deterministically.
- `packages/context/src/ingest/adapters/historic-sql/chunk-unified.ts`
- `packages/context/src/ingest/adapters/historic-sql/chunk-unified.ts`
Keep the same WorkUnits, but mention `emit_historic_sql_evidence` in `notes`.
- `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.ts`
- `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.ts`
Switch production fetch/chunk/scope to the unified hot path, replace skills, remove legacy triage support, and run legacy PGSS baseline cache cleanup.
- `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts`
- `packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts`
Rewrite around unified staging and new skills.
- `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.ts`
- `packages/context/src/ingest/adapters/historic-sql/postgres-pgss-reader.ts`
Inline the PGSS probe logic so `postgres-pgss-query-history-reader.ts` can be deleted.
- `packages/context/src/ingest/local-adapters.ts`
- `packages/context/src/ingest/local-adapters.ts`
Use `PostgresPgssReader` for local Postgres historic SQL and return unified pull config.
- `packages/context/src/ingest/local-bundle-runtime.ts`
- `packages/context/src/ingest/local-bundle-runtime.ts`
Add the source-specific evidence tool to historic-SQL WorkUnits and register the historic-SQL post-processor.
- `packages/context/src/ingest/ingest-runtime-assets.test.ts`
- `packages/context/src/ingest/ingest-runtime-assets.test.ts`
Replace old skill asset assertions with the two new skills.
- `packages/context/src/memory/memory-runtime-assets.test.ts`
- `packages/context/src/memory/memory-runtime-assets.test.ts`
Replace old historic-SQL skill heading with the two new skill headings.
- `packages/context/src/package-exports.test.ts`
- `packages/context/src/package-exports.test.ts`
Remove legacy export assertions and add evidence/projection export assertions.
- `packages/context/src/ingest/index.ts`
- `packages/context/src/ingest/index.ts`
Export new evidence/projection/post-processor helpers and remove legacy historic-SQL exports.
- `packages/cli/src/setup-databases.ts` and `packages/cli/src/historic-sql-doctor.ts`
- `packages/cli/src/setup-databases.ts` and `packages/cli/src/historic-sql-doctor.ts`
Import `PostgresPgssReader` instead of `PostgresPgssQueryHistoryReader`.
- `packages/cli/src/commands/setup-commands.ts`, `packages/cli/src/index.test.ts`, `packages/cli/src/setup-databases.test.ts`
- `packages/cli/src/commands/setup-commands.ts`, `packages/cli/src/index.test.ts`, `packages/cli/src/setup-databases.test.ts`
Rename generated config to `minExecutions` while accepting the old `--historic-sql-min-calls` flag for one release.
- `packages/context/prompts/skills/page_triage_classifier.md`, `packages/context/src/ingest/page-triage/page-triage.service.test.ts`, `packages/context/src/ingest/ingest-prompts.test.ts`
- `packages/context/prompts/skills/page_triage_classifier.md`, `packages/context/src/ingest/page-triage/page-triage.service.test.ts`, `packages/context/src/ingest/ingest-prompts.test.ts`
Remove historic-SQL template triage examples because the new adapter no longer uses page triage.
Delete:

1
examples/local-warehouse/ktx.yaml
View file

@ -2,7 +2,6 @@ project: local-warehouse
connections:
warehouse:
driver: postgres
readonly: true
storage:
state: sqlite
search: sqlite-fts5

1
examples/orbit-relationship-verification/ktx.yaml
View file

@ -3,7 +3,6 @@ connections:
orbit:
driver: sqlite
path: ../../packages/context/test/fixtures/relationship-benchmarks/orbit_style_product_no_declared_constraints/data.sqlite
readonly: true
storage:
state: sqlite
search: sqlite-fts5

11
packages/cli/src/cli-program.ts
View file

@ -178,6 +178,10 @@ function shouldSuppressProjectDirLine(path: string[], options: Record<string, un
return true;
}
if (commandPathKey === 'ktx setup') {
return true;
}
if (
commandPathKey === 'ktx status' &&
typeof options.projectDir !== 'string' &&
@ -357,7 +361,12 @@ export function buildKtxProgram(options: BuildKtxProgramOptions): Command {
registerSetupCommands(program, context);
registerConnectionCommands(program, context);
registerIngestCommands(program, context);
registerIngestCommands(program, context, {
runTextIngest: async (textIngestArgs, ingestIo, ingestDeps) => {
const { runKtxTextIngest } = await import('./text-ingest.js');
return await (ingestDeps.textIngest ?? runKtxTextIngest)(textIngestArgs, ingestIo);
},
});
registerWikiCommands(program, context);
registerSlCommands(program, context);
registerStatusCommands(program, context);

2
packages/cli/src/cli-runtime.ts
View file

@ -8,6 +8,7 @@ import type { KtxRuntimeArgs } from './runtime.js';
import type { KtxSetupArgs } from './setup.js';
import type { KtxSlArgs } from './sl.js';
import { profileMark, profileSpan } from './startup-profile.js';
import type { KtxTextIngestArgs } from './text-ingest.js';
profileMark('module:cli-runtime');
@ -29,6 +30,7 @@ export interface KtxCliDeps {
connection?: (args: KtxConnectionArgs, io: KtxCliIo) => Promise<number>;
doctor?: (args: KtxDoctorArgs, io: KtxCliIo) => Promise<number>;
publicIngest?: (args: KtxPublicIngestArgs, io: KtxCliIo) => Promise<number>;
textIngest?: (args: KtxTextIngestArgs, io: KtxCliIo) => Promise<number>;
runtime?: (args: KtxRuntimeArgs, io: KtxCliIo) => Promise<number>;
knowledge?: (args: KtxKnowledgeArgs, io: KtxCliIo) => Promise<number>;
sl?: (args: KtxSlArgs, io: KtxCliIo) => Promise<number>;

41
packages/cli/src/commands/ingest-commands.ts
View file

@ -1,16 +1,27 @@
import { type Command, Option } from '@commander-js/extra-typings';
import {
collectOption,
type KtxCliCommandContext,
parsePositiveIntegerOption,
resolveCommandProjectDir,
} from '../cli-program.js';
import type { KtxCliDeps, KtxCliIo } from '../index.js';
import { runtimeInstallPolicyFromFlags } from '../managed-python-command.js';
import type { KtxPublicIngestArgs } from '../public-ingest.js';
import { profileMark } from '../startup-profile.js';
import type { KtxTextIngestArgs } from '../text-ingest.js';
profileMark('module:commands/ingest-commands');
export function registerIngestCommands(program: Command, context: KtxCliCommandContext): void {
interface IngestCommandOptions {
runTextIngest: (args: KtxTextIngestArgs, io: KtxCliIo, deps: KtxCliDeps) => Promise<number>;
}
export function registerIngestCommands(
program: Command,
context: KtxCliCommandContext,
commandOptions: IngestCommandOptions,
): void {
const ingest = program
.command('ingest')
.description('Build or inspect KTX context')
@ -51,4 +62,32 @@ export function registerIngestCommands(program: Command, context: KtxCliCommandC
ingest.hook('preAction', (_thisCommand, actionCommand) => {
context.writeDebug?.('ingest', actionCommand);
});
ingest
.command('text')
.description('Ingest free-form text artifacts into KTX memory')
.argument('[files...]', 'Files to ingest; use - to read one item from stdin')
.option('--text <content>', 'Text content to ingest; repeat for a batch', collectOption, [])
.option('--connection-id <connectionId>', 'Optional KTX connection id for semantic-layer capture')
.option('--user-id <id>', 'Memory user id for capture attribution', 'local-cli')
.option('--json', 'Print JSON output')
.option('--fail-fast', 'Stop after the first failed text item', false)
.action(async (files: string[], options, command) => {
const parentOptions = command.parent?.opts() as { json?: boolean } | undefined;
context.setExitCode(
await commandOptions.runTextIngest(
{
projectDir: resolveCommandProjectDir(command),
texts: options.text,
files,
...(options.connectionId ? { connectionId: options.connectionId } : {}),
userId: options.userId,
json: options.json === true || parentOptions?.json === true,
failFast: options.failFast === true,
},
context.io,
context.deps,
),
);
});
}

217
packages/cli/src/commands/setup-commands.ts
View file

@ -64,13 +64,6 @@ function sourceType(value: string): KtxSetupSourceType {
throw new InvalidArgumentError(`invalid choice '${value}'`);
}
function agentScope(value: string): 'project' | 'global' {
if (value === 'project' || value === 'global') {
return value;
}
throw new InvalidArgumentError(`invalid choice '${value}'`);
}
function positiveNumber(value: string): number {
const parsed = Number.parseInt(value, 10);
if (!Number.isInteger(parsed) || parsed <= 0) {
@ -97,7 +90,6 @@ function shouldShowSetupEntryMenu(
agents?: boolean;
target?: string;
global?: boolean;
project?: boolean;
skipAgents?: boolean;
yes?: boolean;
input?: boolean;
@ -142,7 +134,6 @@ function shouldShowSetupEntryMenu(
metabaseDatabaseId?: number;
notionCrawlMode?: string;
notionRootPageId?: string[];
skipInitialSourceIngest?: boolean;
skipSources?: boolean;
},
command: Command,
@ -172,7 +163,6 @@ function shouldShowSetupEntryMenu(
'agents',
'target',
'global',
'project',
'skipAgents',
'yes',
'input',
@ -211,7 +201,6 @@ function shouldShowSetupEntryMenu(
'sourceTarget',
'metabaseDatabaseId',
'notionCrawlMode',
'skipInitialSourceIngest',
'skipSources',
].some((optionName) => optionWasSpecified(command, optionName));
}
@ -220,9 +209,9 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo
const setup = program
.command('setup')
.description('Set up or resume a local KTX project')
.option('--project-dir <path>', 'KTX project directory')
.option('--new', 'Create a new KTX project before setup', false)
.option('--existing', 'Use an existing KTX project', false)
.addOption(new Option('--project-dir <path>', 'KTX project directory').hideHelp())
.addOption(new Option('--new', 'Create a new KTX project before setup').hideHelp().default(false))
.addOption(new Option('--existing', 'Use an existing KTX project').hideHelp().default(false))
.option('--agents', 'Install agent integration only', false)
.addOption(
new Option('--target <target>', 'Agent target').choices([
@ -233,94 +222,128 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo
'universal',
]),
)
.addOption(new Option('--agent-scope <scope>', 'Agent install scope').argParser(agentScope).default('project'))
.option('--project', 'Install agent integration into the project scope', false)
.option('--global', 'Install agent integration into the global target scope', false)
.option('--skip-agents', 'Leave agent integration incomplete for now', false)
.addOption(new Option('--skip-agents', 'Leave agent integration incomplete for now').hideHelp().default(false))
.option('--yes', 'Accept safe defaults in non-interactive setup', false)
.option('--no-input', 'Disable interactive terminal input')
.addOption(new Option('--llm-backend <backend>', 'LLM backend').argParser(llmBackend))
.option('--anthropic-api-key-env <name>', 'Environment variable containing the Anthropic API key')
.option('--anthropic-api-key-file <path>', 'File containing the Anthropic API key')
.option('--anthropic-model <model>', 'Anthropic model ID to validate and save')
.option('--vertex-project <project>', 'Google Vertex AI project ID, env:NAME, or file:/path')
.option('--vertex-location <location>', 'Google Vertex AI location, env:NAME, or file:/path')
.addOption(new Option('--skip-llm', 'Leave LLM setup incomplete for now').hideHelp().default(false))
.addOption(new Option('--embedding-backend <backend>', 'Embedding backend').argParser(embeddingBackend))
.option('--embedding-api-key-env <name>', 'Environment variable containing the embedding provider API key')
.option('--embedding-api-key-file <path>', 'File containing the embedding provider API key')
.addOption(new Option('--skip-embeddings', 'Leave embedding setup incomplete for now').hideHelp().default(false))
.option(
'--database <driver>',
'Database driver to configure; repeatable',
(value, previous: KtxSetupDatabaseDriver[]) => {
return [...previous, databaseDriver(value)];
},
[] as KtxSetupDatabaseDriver[],
)
.option(
'--database-connection-id <id>',
'Existing selected connection id or new connection id',
(value, previous: string[]) => [...previous, value],
[],
)
.option('--new-database-connection-id <id>', 'Connection id for one new database connection', (value) => {
if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(value)) {
throw new InvalidArgumentError(`Unsafe connection id: ${value}`);
}
return value;
})
.option('--database-url <url>', 'URL, env:NAME, or file:/path for one new URL-style database connection')
.option(
'--database-schema <schema>',
'Database schema to include; repeatable',
(value, previous: string[]) => [...previous, value],
[],
)
.option('--enable-query-history', 'Enable query history when the selected database supports it', false)
.option('--disable-query-history', 'Disable query history for the selected database', false)
.option('--query-history-window-days <number>', 'Query-history lookback window', positiveInteger)
.option('--query-history-min-executions <number>', 'Minimum executions for a query-history template', positiveInteger)
.option(
'--query-history-service-account-pattern <pattern>',
'Query-history service-account regex; repeatable',
(value, previous: string[]) => [...previous, value],
[],
)
.option(
'--query-history-redaction-pattern <pattern>',
'Query-history SQL-literal redaction regex; repeatable',
(value, previous: string[]) => [...previous, value],
[],
)
.option('--skip-databases', 'Leave database setup incomplete; KTX cannot work until a database is added', false)
.addOption(new Option('--source <type>', 'Source connector type').argParser(sourceType))
.option('--source-connection-id <id>', 'Connection id for source setup')
.option('--source-path <path>', 'Local source path for dbt, MetricFlow, or LookML')
.option('--source-git-url <url>', 'Git URL for dbt, MetricFlow, or LookML')
.option('--source-branch <branch>', 'Git branch for source setup')
.option('--source-subpath <path>', 'Repo subpath for source setup')
.option('--source-auth-token-ref <ref>', 'env: or file: credential ref for source repo auth')
.option('--source-url <url>', 'Source service URL for Metabase or Looker')
.option('--source-api-key-ref <ref>', 'env: or file: API key ref for Metabase or Notion')
.option('--source-client-id <id>', 'Looker client id')
.option('--source-client-secret-ref <ref>', 'env: or file: Looker client secret ref')
.option('--source-warehouse-connection-id <id>', 'Mapped warehouse connection id')
.option('--source-project-name <name>', 'dbt project name override')
.option('--source-profiles-path <path>', 'dbt profiles path')
.option('--source-target <target>', 'dbt target or source-specific mapping target')
.option('--metabase-database-id <id>', 'Metabase database id to map', positiveNumber)
.addOption(new Option('--llm-backend <backend>', 'LLM backend').argParser(llmBackend).hideHelp())
.addOption(
new Option('--notion-crawl-mode <mode>', 'Notion crawl mode').choices(['all_accessible', 'selected_roots']),
new Option('--anthropic-api-key-env <name>', 'Environment variable containing the Anthropic API key').hideHelp(),
)
.option(
'--notion-root-page-id <id>',
'Notion root page id; repeatable',
(value, previous: string[]) => [...previous, value],
[],
.addOption(
new Option('--anthropic-api-key-file <path>', 'File containing the Anthropic API key').hideHelp(),
)
.option('--skip-initial-source-ingest', 'Validate source setup without building source context during setup', false)
.option('--skip-sources', 'Mark optional source setup complete with no sources', false)
.addOption(new Option('--anthropic-model <model>', 'Anthropic model ID to validate and save').hideHelp())
.addOption(new Option('--vertex-project <project>', 'Google Vertex AI project ID, env:NAME, or file:/path').hideHelp())
.addOption(new Option('--vertex-location <location>', 'Google Vertex AI location, env:NAME, or file:/path').hideHelp())
.addOption(new Option('--skip-llm', 'Leave LLM setup incomplete for now').hideHelp().default(false))
.addOption(new Option('--embedding-backend <backend>', 'Embedding backend').argParser(embeddingBackend).hideHelp())
.addOption(
new Option(
'--embedding-api-key-env <name>',
'Environment variable containing the embedding provider API key',
).hideHelp(),
)
.addOption(
new Option('--embedding-api-key-file <path>', 'File containing the embedding provider API key').hideHelp(),
)
.addOption(new Option('--skip-embeddings', 'Leave embedding setup incomplete for now').hideHelp().default(false))
.addOption(
new Option('--database <driver>', 'Database driver to configure; repeatable')
.argParser((value, previous: KtxSetupDatabaseDriver[]) => {
return [...previous, databaseDriver(value)];
})
.default([] as KtxSetupDatabaseDriver[])
.hideHelp(),
)
.addOption(
new Option('--database-connection-id <id>', 'Existing selected connection id or new connection id')
.argParser((value, previous: string[]) => [...previous, value])
.default([] as string[])
.hideHelp(),
)
.addOption(
new Option('--new-database-connection-id <id>', 'Connection id for one new database connection')
.argParser((value) => {
if (!/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/.test(value)) {
throw new InvalidArgumentError(`Unsafe connection id: ${value}`);
}
return value;
})
.hideHelp(),
)
.addOption(
new Option('--database-url <url>', 'URL, env:NAME, or file:/path for one new URL-style database connection').hideHelp(),
)
.addOption(
new Option('--database-schema <schema>', 'Database schema to include; repeatable')
.argParser((value, previous: string[]) => [...previous, value])
.default([] as string[])
.hideHelp(),
)
.addOption(
new Option('--enable-query-history', 'Enable query history when the selected database supports it')
.hideHelp()
.default(false),
)
.addOption(
new Option('--disable-query-history', 'Disable query history for the selected database').hideHelp().default(false),
)
.addOption(
new Option('--query-history-window-days <number>', 'Query-history lookback window')
.argParser(positiveInteger)
.hideHelp(),
)
.addOption(
new Option('--query-history-min-executions <number>', 'Minimum executions for a query-history template')
.argParser(positiveInteger)
.hideHelp(),
)
.addOption(
new Option('--query-history-service-account-pattern <pattern>', 'Query-history service-account regex; repeatable')
.argParser((value, previous: string[]) => [...previous, value])
.default([] as string[])
.hideHelp(),
)
.addOption(
new Option('--query-history-redaction-pattern <pattern>', 'Query-history SQL-literal redaction regex; repeatable')
.argParser((value, previous: string[]) => [...previous, value])
.default([] as string[])
.hideHelp(),
)
.addOption(
new Option('--skip-databases', 'Leave database setup incomplete; KTX cannot work until a database is added')
.hideHelp()
.default(false),
)
.addOption(new Option('--source <type>', 'Source connector type').argParser(sourceType).hideHelp())
.addOption(new Option('--source-connection-id <id>', 'Connection id for source setup').hideHelp())
.addOption(new Option('--source-path <path>', 'Local source path for dbt, MetricFlow, or LookML').hideHelp())
.addOption(new Option('--source-git-url <url>', 'Git URL for dbt, MetricFlow, or LookML').hideHelp())
.addOption(new Option('--source-branch <branch>', 'Git branch for source setup').hideHelp())
.addOption(new Option('--source-subpath <path>', 'Repo subpath for source setup').hideHelp())
.addOption(new Option('--source-auth-token-ref <ref>', 'env: or file: credential ref for source repo auth').hideHelp())
.addOption(new Option('--source-url <url>', 'Source service URL for Metabase or Looker').hideHelp())
.addOption(new Option('--source-api-key-ref <ref>', 'env: or file: API key ref for Metabase or Notion').hideHelp())
.addOption(new Option('--source-client-id <id>', 'Looker client id').hideHelp())
.addOption(new Option('--source-client-secret-ref <ref>', 'env: or file: Looker client secret ref').hideHelp())
.addOption(new Option('--source-warehouse-connection-id <id>', 'Mapped warehouse connection id').hideHelp())
.addOption(new Option('--source-project-name <name>', 'dbt project name override').hideHelp())
.addOption(new Option('--source-profiles-path <path>', 'dbt profiles path').hideHelp())
.addOption(new Option('--source-target <target>', 'dbt target or source-specific mapping target').hideHelp())
.addOption(new Option('--metabase-database-id <id>', 'Metabase database id to map').argParser(positiveNumber).hideHelp())
.addOption(
new Option('--notion-crawl-mode <mode>', 'Notion crawl mode')
.choices(['all_accessible', 'selected_roots'])
.hideHelp(),
)
.addOption(
new Option('--notion-root-page-id <id>', 'Notion root page id; repeatable')
.argParser((value, previous: string[]) => [...previous, value])
.default([] as string[])
.hideHelp(),
)
.addOption(new Option('--skip-sources', 'Mark optional source setup complete with no sources').hideHelp().default(false))
.showHelpAfterError();
setup.hook('preAction', (_thisCommand, actionCommand) => {
@ -371,7 +394,7 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo
}
const mode = options.new ? 'new' : options.existing ? 'existing' : 'auto';
const resolvedAgentScope = options.global ? 'global' : options.agentScope;
const resolvedAgentScope = options.global ? 'global' : 'project';
await runSetupArgs(context, {
command: 'run',
projectDir: resolveCommandProjectDir(command),

8
packages/cli/src/connection.test.ts
View file

@ -94,7 +94,7 @@ describe('runKtxConnection', () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
await writeConnections(projectDir, {
warehouse: { driver: 'postgres', url: 'env:DATABASE_URL', readonly: true },
warehouse: { driver: 'postgres', url: 'env:DATABASE_URL' },
docs: { driver: 'notion', auth_token_ref: 'env:NOTION_TOKEN', crawl_mode: 'all_accessible' },
});
const io = makeIo();
@ -123,7 +123,7 @@ describe('runKtxConnection', () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
await writeConnections(projectDir, {
warehouse: { driver: 'sqlite', readonly: true },
warehouse: { driver: 'sqlite' },
});
const { connector, introspect, cleanup } = nativeConnector('sqlite', ['customers', 'orders']);
const createScanConnector = vi.fn(async () => connector);
@ -159,7 +159,7 @@ describe('runKtxConnection', () => {
prod_metabase: {
driver: 'metabase',
api_url: 'http://metabase.example.test',
api_key: 'mb_test',
api_key: 'mb_test', // pragma: allowlist secret
},
});
const testConnection = vi.fn(async () => ({ success: true as const }));
@ -202,7 +202,7 @@ describe('runKtxConnection', () => {
const projectDir = join(tempDir, 'project');
await initKtxProject({ projectDir, projectName: 'warehouse' });
await writeConnections(projectDir, {
warehouse: { driver: 'sqlite', readonly: true },
warehouse: { driver: 'sqlite' },
});
const cleanup = vi.fn(async () => undefined);
const connector: KtxScanConnector = {

24
packages/cli/src/context-build-view.test.ts
View file

@ -175,6 +175,30 @@ describe('renderContextBuildView', () => {
expect(output).toContain('dbt-main');
});
it('supports text ingest labels while preserving the shared compact progress view', () => {
const state = initViewState([
{ connectionId: 'text-1', driver: 'text', operation: 'source-ingest', debugCommand: '', steps: ['memory-update'] },
{ connectionId: 'schema.md', driver: 'text', operation: 'source-ingest', debugCommand: '', steps: ['memory-update'] },
]);
state.contextSources[0].status = 'running';
state.contextSources[0].detailLine = 'capturing...';
const output = renderContextBuildView(state, {
styled: false,
title: 'Ingesting text memory',
contextGroupLabel: 'Texts',
sourceIngestRunningText: 'capturing...',
completedItemName: { singular: 'text', plural: 'texts' },
});
expect(output).toContain('Ingesting text memory');
expect(output).toContain('Texts:');
expect(output).toContain('text-1');
expect(output).toContain('schema.md');
expect(output).toContain('capturing...');
expect(output).not.toContain('Context sources:');
});
it('renders header with total elapsed time when set', () => {
const state = initViewState([
{ connectionId: 'warehouse', driver: 'postgres', operation: 'database-ingest', debugCommand: '', steps: ['database-schema'] },

57
packages/cli/src/context-build-view.ts
View file

@ -115,6 +115,26 @@ export interface ContextBuildSourceProgressUpdate {
summaryText?: string;
}
interface CompletedItemName {
singular: string;
plural: string;
}
interface ContextBuildRenderOptions {
styled?: boolean;
showHint?: boolean;
hintText?: string;
projectDir?: string;
title?: string;
primaryGroupLabel?: string;
contextGroupLabel?: string;
scanRunningText?: string;
sourceIngestRunningText?: string;
completedItemName?: CompletedItemName;
notices?: string[];
warnings?: string[];
}
export interface ContextBuildDeps {
executeTarget?: typeof executePublicIngestTarget;
now?: () => number;
@ -224,7 +244,7 @@ function staleProgressText(target: ContextBuildTargetState, styled: boolean): st
return styled ? dim(text) : text;
}
function targetDetail(target: ContextBuildTargetState, styled: boolean): string {
function targetDetail(target: ContextBuildTargetState, styled: boolean, options: ContextBuildRenderOptions): string {
if (target.status === 'done') {
const parts: string[] = [];
if (target.summaryText) parts.push(target.summaryText);
@ -239,7 +259,9 @@ function targetDetail(target: ContextBuildTargetState, styled: boolean): string
const percent = extractPercent(target.detailLine);
const progressText =
target.detailLine?.replace(/^\[\d+%\]\s*/, '') ??
(target.target.operation === 'database-ingest' ? 'reading schema' : 'ingesting...');
(target.target.operation === 'database-ingest'
? (options.scanRunningText ?? 'reading schema')
: (options.sourceIngestRunningText ?? 'ingesting...'));
const elapsed = target.elapsedMs > 0 ? `(${formatDuration(target.elapsedMs)})` : null;
const parts: string[] = [];
if (percent !== null) {
@ -308,7 +330,13 @@ function columnWidth(state: ContextBuildViewState): number {
return Math.max(12, ...all.map((t) => t.target.connectionId.length)) + 2;
}
function renderTargetRows(target: ContextBuildTargetState, frame: number, styled: boolean, width: number): string[] {
function renderTargetRows(
target: ContextBuildTargetState,
frame: number,
styled: boolean,
width: number,
options: ContextBuildRenderOptions,
): string[] {
const icon = statusIcon(target.status, frame, styled);
const name = target.target.connectionId.padEnd(width);
const anyPhaseStarted = target.phases.some((p) => p.status !== 'queued');
@ -317,7 +345,7 @@ function renderTargetRows(target: ContextBuildTargetState, frame: number, styled
const headerLine = ` ${icon} ${name} ${headerDetail}`.trimEnd();
return [headerLine, ...target.phases.map((phase) => renderPhaseRow(phase, frame, styled))];
}
return [` ${icon} ${name} ${targetDetail(target, styled)}`];
return [` ${icon} ${name} ${targetDetail(target, styled, options)}`];
}
function renderTargetGroup(
@ -326,9 +354,10 @@ function renderTargetGroup(
frame: number,
styled: boolean,
width: number,
options: ContextBuildRenderOptions,
): string[] {
if (targets.length === 0) return [];
return ['', ` ${label}:`, ...targets.flatMap((t) => renderTargetRows(t, frame, styled, width))];
return ['', ` ${label}:`, ...targets.flatMap((t) => renderTargetRows(t, frame, styled, width, options))];
}
function renderMessageGroup(label: string, messages: string[], styled: boolean): string[] {
@ -360,14 +389,7 @@ function retryCommand(input: {
export function renderContextBuildView(
state: ContextBuildViewState,
options: {
styled?: boolean;
showHint?: boolean;
hintText?: string;
projectDir?: string;
notices?: string[];
warnings?: string[];
} = {},
options: ContextBuildRenderOptions = {},
): string {
const styled = options.styled ?? true;
const width = columnWidth(state);
@ -377,7 +399,7 @@ export function renderContextBuildView(
const hasActive = allTargets.some((t) => t.status === 'running' || t.status === 'queued');
const allDone = totalCount > 0 && !hasActive;
const headerParts = ['Building KTX context'];
const headerParts = [options.title ?? 'Building KTX context'];
if (totalCount > 0) {
const progressParts: string[] = [`${doneCount}/${totalCount}`];
if (state.totalElapsedMs > 0) progressParts.push(formatDuration(state.totalElapsedMs));
@ -393,15 +415,16 @@ export function renderContextBuildView(
header,
separator,
...(options.projectDir ? [` Project: ${options.projectDir}`] : []),
...renderTargetGroup('Databases', state.primarySources, state.frame, styled, width),
...renderTargetGroup('Context sources', state.contextSources, state.frame, styled, width),
...renderTargetGroup(options.primaryGroupLabel ?? 'Databases', state.primarySources, state.frame, styled, width, options),
...renderTargetGroup(options.contextGroupLabel ?? 'Context sources', state.contextSources, state.frame, styled, width, options),
...renderMessageGroup('Notices', options.notices ?? [], styled),
...renderMessageGroup('Warnings', options.warnings ?? [], styled),
'',
];
if (allDone && state.totalElapsedMs > 0) {
const sourcesLabel = totalCount === 1 ? '1 source' : `${totalCount} sources`;
const itemName = options.completedItemName ?? { singular: 'source', plural: 'sources' };
const sourcesLabel = totalCount === 1 ? `1 ${itemName.singular}` : `${totalCount} ${itemName.plural}`;
const summary = ` Done in ${formatDuration(state.totalElapsedMs)} · ${sourcesLabel} processed`;
lines.push(styled ? green(summary) : summary);
lines.push('');

1
packages/cli/src/demo-assets.ts
View file

@ -57,7 +57,6 @@ function demoConfig(databasePath: string): string {
` ${DEMO_CONNECTION_ID}:`,
' driver: sqlite',
` path: ${JSON.stringify(databasePath)}`,
' readonly: true',
'storage:',
' state: sqlite',
' search: sqlite-fts5',

1
packages/cli/src/doctor.test.ts
View file

@ -275,7 +275,6 @@ describe('runKtxDoctor', () => {
' warehouse:',
' driver: postgres',
' url: env:WAREHOUSE_DATABASE_URL',
' readonly: true',
' context:',
' queryHistory:',
' enabled: true',

8
packages/cli/src/historic-sql-doctor.test.ts
View file

@ -25,7 +25,7 @@ describe('runPostgresHistoricSqlDoctorChecks', () => {
it('passes when no Postgres query-history connections are enabled', async () => {
const checks = await runPostgresHistoricSqlDoctorChecks(
projectWithConnections({
warehouse: { driver: 'sqlite', path: './warehouse.db', readonly: true },
warehouse: { driver: 'sqlite', path: './warehouse.db' },
}),
{
postgresHistoricSqlProbe: vi.fn<PostgresHistoricSqlDoctorProbe>(),
@ -53,7 +53,6 @@ describe('runPostgresHistoricSqlDoctorChecks', () => {
warehouse: {
driver: 'postgres',
url: 'env:WAREHOUSE_DATABASE_URL',
readonly: true,
context: { queryHistory: { enabled: true } },
},
}),
@ -66,7 +65,6 @@ describe('runPostgresHistoricSqlDoctorChecks', () => {
connection: {
driver: 'postgres',
url: 'env:WAREHOUSE_DATABASE_URL',
readonly: true,
context: { queryHistory: { enabled: true } },
},
env: process.env,
@ -87,7 +85,6 @@ describe('runPostgresHistoricSqlDoctorChecks', () => {
warehouse: {
driver: 'postgres',
url: 'env:WAREHOUSE_DATABASE_URL',
readonly: true,
context: { queryHistory: { enabled: true } },
},
}),
@ -119,7 +116,6 @@ describe('runPostgresHistoricSqlDoctorChecks', () => {
warehouse: {
driver: 'postgres',
url: 'env:WAREHOUSE_DATABASE_URL',
readonly: true,
context: { queryHistory: { enabled: true } },
},
}),
@ -182,7 +178,6 @@ describe('runPostgresHistoricSqlDoctorChecks', () => {
warehouse: {
driver: 'mysql',
url: 'env:WAREHOUSE_DATABASE_URL',
readonly: true,
context: { queryHistory: { enabled: true } },
},
}),
@ -208,7 +203,6 @@ describe('runPostgresHistoricSqlDoctorChecks', () => {
warehouse: {
driver: 'postgres',
url: 'env:WAREHOUSE_DATABASE_URL',
readonly: true,
context: { queryHistory: { enabled: true } },
},
}),

3
packages/cli/src/historic-sql-doctor.ts
View file

@ -96,8 +96,9 @@ async function defaultPostgresHistoricSqlProbe(
const [{ PostgresPgssReader }, { KtxPostgresHistoricSqlQueryClient, isKtxPostgresConnectionConfig }] =
await Promise.all([import('@ktx/context/ingest'), import('@ktx/connector-postgres')]);
const inputDriver = input.connection.driver ?? 'unknown';
if (!isKtxPostgresConnectionConfig(input.connection)) {
throw new Error(`Native PostgreSQL connector cannot run driver "${input.connection.driver ?? 'unknown'}"`);
throw new Error(`Native PostgreSQL connector cannot run driver "${inputDriver}"`);
}
const client = new KtxPostgresHistoricSqlQueryClient({

156
packages/cli/src/index.test.ts
View file

@ -323,6 +323,22 @@ describe('runKtxCli', () => {
expect(testIo.stderr()).toBe(`Project: ${tempDir}\n`);
});
it('does not print the command-level project directory line for setup', async () => {
const setup = vi.fn(async () => 0);
const testIo = makeIo();
await expect(runKtxCli(['--project-dir', tempDir, 'setup', '--no-input'], testIo.io, { setup })).resolves.toBe(0);
expect(setup).toHaveBeenCalledWith(
expect.objectContaining({
command: 'run',
projectDir: tempDir,
}),
testIo.io,
);
expect(testIo.stderr()).toBe('');
});
it('skips the project directory line for JSON output mode', async () => {
const publicIngest = vi.fn(async () => 0);
const jsonIo = makeIo();
@ -410,35 +426,62 @@ describe('runKtxCli', () => {
expect(io.stderr()).toContain('Choose only one runtime install mode: --yes or --no-input');
});
it('documents setup as a bare command without subcommands', async () => {
it('documents setup with only the common interactive options visible', async () => {
const testIo = makeIo();
await expect(runKtxCli(['setup', '--help'], testIo.io)).resolves.toBe(0);
expect(testIo.stdout()).toContain('Usage: ktx setup [options]');
expect(testIo.stdout()).not.toContain('Commands:');
expect(testIo.stdout()).not.toContain('setup demo');
expect(testIo.stdout()).not.toContain('setup context');
expect(testIo.stdout()).not.toContain('--skip-llm');
expect(testIo.stdout()).not.toContain('--skip-embeddings');
expect(testIo.stdout()).not.toContain('--embedding-model');
expect(testIo.stdout()).not.toContain('--embedding-dimensions');
expect(testIo.stdout()).not.toContain('--embedding-base-url');
for (const expected of [
const stdout = testIo.stdout();
expect(stdout).toContain('Usage: ktx setup [options]');
expect(stdout).toContain('--agents');
expect(stdout).toContain('--target <target>');
expect(stdout).toContain('--global');
expect(stdout).toContain('--yes');
expect(stdout).toContain('--no-input');
expect(stdout).toContain('Global Options:');
expect(stdout.match(/--project-dir <path>/g)).toHaveLength(1);
expect(stdout).not.toContain('Commands:');
expect(stdout).not.toContain('setup demo');
expect(stdout).not.toContain('setup context');
for (const hiddenFlag of [
'--new',
'--existing',
'--agent-scope',
'--skip-agents',
'--llm-backend',
'--anthropic-api-key-env',
'--vertex-project',
'--embedding-backend',
'--database ',
'--database-connection-id',
'--new-database-connection-id',
'--enable-historic-sql',
'--historic-sql-min-executions',
'--enable-query-history',
'--disable-query-history',
'--query-history-window-days',
'--query-history-min-executions',
'--query-history-service-account-pattern',
'--query-history-redaction-pattern',
'--skip-databases',
'--source ',
'--source-connection-id',
'--metabase-database-id',
'--notion-root-page-id',
'--skip-initial-source-ingest',
'--skip-sources',
'--skip-llm',
'--skip-embeddings',
'--embedding-model',
'--embedding-dimensions',
'--embedding-base-url',
]) {
expect(testIo.stdout()).toContain(expected);
expect(stdout).not.toContain(hiddenFlag);
}
expect(testIo.stdout()).toContain('KTX cannot work until a database is added');
expect(testIo.stdout()).not.toContain('primary ' + 'source');
expect(testIo.stdout()).not.toContain('primary ' + 'sources');
expect(testIo.stdout()).not.toContain('--enable-historic-sql');
expect(testIo.stdout()).not.toContain('--historic-sql-window-days');
expect(stdout).not.toMatch(/^ --project\s/m);
expect(stdout).not.toContain('primary ' + 'source');
expect(stdout).not.toContain('primary ' + 'sources');
expect(testIo.stderr()).toBe('');
});
@ -737,13 +780,30 @@ describe('runKtxCli', () => {
expect(setup).not.toHaveBeenCalled();
});
it('rejects removed setup options', async () => {
const setup = vi.fn(async () => 0);
const cases = [
['setup', '--project'],
['setup', '--agent-scope', 'global'],
['setup', '--skip-initial-source-ingest'],
];
for (const args of cases) {
const testIo = makeIo();
await expect(runKtxCli(['--project-dir', tempDir, ...args], testIo.io, { setup })).resolves.toBe(1);
expect(testIo.stderr()).toMatch(/unknown option|error:/i);
}
expect(setup).not.toHaveBeenCalled();
});
it('prints ingest help without invoking ingest execution', async () => {
const testIo = makeIo();
const publicIngest = vi.fn();
await expect(runKtxCli(['ingest', '--help'], testIo.io, { publicIngest })).resolves.toBe(0);
expect(testIo.stdout()).toContain('Usage: ktx ingest [options] [connectionId]');
expect(testIo.stdout()).toContain('Usage: ktx ingest');
expect(testIo.stdout()).toContain('Build or inspect KTX context');
expect(testIo.stdout()).toContain('--all');
expect(testIo.stdout()).toContain('--fast');
@ -751,14 +811,73 @@ describe('runKtxCli', () => {
expect(testIo.stdout()).toContain('--query-history');
expect(testIo.stdout()).toContain('--no-query-history');
expect(testIo.stdout()).toContain('--query-history-window-days <days>');
expect(testIo.stdout()).toContain('text');
expect(testIo.stdout()).not.toMatch(/^ status\s/m);
expect(testIo.stdout()).not.toMatch(/^ replay\s/m);
expect(testIo.stdout()).not.toMatch(/^ run\s/m);
expect(testIo.stdout()).not.toMatch(/^ watch\s/m);
expect(testIo.stdout()).not.toContain('--manifest');
expect(testIo.stderr()).toBe('');
expect(publicIngest).not.toHaveBeenCalled();
});
it('routes text memory ingest through Commander without exposing chat ids', async () => {
const textIngest = vi.fn(async () => 0);
const testIo = makeIo();
await expect(
runKtxCli(
[
'--project-dir',
tempDir,
'ingest',
'text',
'--text',
'Revenue means gross receipts.',
'--text',
'Orders are completed purchases.',
'--connection-id',
'warehouse',
'--user-id',
'agent',
'--json',
'--fail-fast',
],
testIo.io,
{ textIngest },
),
).resolves.toBe(0);
expect(textIngest).toHaveBeenCalledWith(
{
projectDir: tempDir,
texts: ['Revenue means gross receipts.', 'Orders are completed purchases.'],
files: [],
connectionId: 'warehouse',
userId: 'agent',
json: true,
failFast: true,
},
testIo.io,
);
expect(testIo.stderr()).toBe('');
});
it('documents text ingest inputs without a manifest option', async () => {
const textIngest = vi.fn(async () => 0);
const testIo = makeIo();
await expect(runKtxCli(['ingest', 'text', '--help'], testIo.io, { textIngest })).resolves.toBe(0);
expect(testIo.stdout()).toContain('Usage: ktx ingest text [options] [files...]');
expect(testIo.stdout()).toContain('--text <content>');
expect(testIo.stdout()).toContain('--connection-id <connectionId>');
expect(testIo.stdout()).toContain('--user-id <id>');
expect(testIo.stdout()).toContain('--fail-fast');
expect(testIo.stdout()).not.toContain('--manifest');
expect(textIngest).not.toHaveBeenCalled();
});
it('rejects old adapter-backed ingest flags at the top level and under dev', async () => {
const rootRunIo = makeIo();
const devRunIo = makeIo();
@ -1142,7 +1261,6 @@ describe('runKtxCli', () => {
'--agents',
'--target',
'codex',
'--project',
'--no-input',
'--yes',
],

3
packages/cli/src/local-adapters.test.ts
View file

@ -45,7 +45,6 @@ describe('CLI local ingest adapters', () => {
' warehouse:',
' driver: postgres',
' url: env:WAREHOUSE_DATABASE_URL',
' readonly: true',
' historicSql:',
' enabled: true',
' dialect: postgres',
@ -108,7 +107,6 @@ describe('CLI local ingest adapters', () => {
'connections:',
' bq:',
' driver: bigquery',
' readonly: true',
' dataset_id: analytics',
' location: us',
' credentials_json: \'{"project_id":"demo-project"}\'',
@ -142,7 +140,6 @@ describe('CLI local ingest adapters', () => {
'connections:',
' sf:',
' driver: snowflake',
' readonly: true',
' account: acct',
' warehouse: wh',
' database: ANALYTICS',

20
packages/cli/src/local-adapters.ts
View file

@ -208,10 +208,9 @@ function enabledHistoricSqlDialect(connection: unknown): 'postgres' | 'bigquery'
function createEphemeralPostgresHistoricSqlClient(project: KtxLocalProject, connectionId: string) {
const connection = project.config.connections[connectionId] as KtxPostgresConnectionConfig | undefined;
const inputDriver = connection?.driver ?? 'unknown';
if (!isKtxPostgresConnectionConfig(connection)) {
throw new Error(
`Query history ingest requires a Postgres connection, got ${String(connection?.driver ?? 'unknown')}`,
);
throw new Error(`Query history ingest requires a Postgres connection, got ${String(inputDriver)}`);
}
return {
async executeQuery(sql: string, params?: unknown[]) {
@ -230,10 +229,9 @@ function createEphemeralPostgresHistoricSqlClient(project: KtxLocalProject, conn
function createEphemeralBigQueryHistoricSqlClient(project: KtxLocalProject, connectionId: string) {
const connection = project.config.connections[connectionId] as KtxBigQueryConnectionConfig | undefined;
const inputDriver = connection?.driver ?? 'unknown';
if (!isKtxBigQueryConnectionConfig(connection)) {
throw new Error(
`Query history ingest requires a BigQuery connection, got ${String(connection?.driver ?? 'unknown')}`,
);
throw new Error(`Query history ingest requires a BigQuery connection, got ${String(inputDriver)}`);
}
return {
async executeQuery(query: string) {
@ -261,10 +259,9 @@ async function createEphemeralSnowflakeHistoricSqlClient(
connectorModule: SnowflakeConnectorModule,
) {
const connection = project.config.connections[connectionId];
const inputDriver = connection?.driver ?? 'unknown';
if (!connectorModule.isKtxSnowflakeConnectionConfig(connection)) {
throw new Error(
`Query history ingest requires a Snowflake connection, got ${String(connection?.driver ?? 'unknown')}`,
);
throw new Error(`Query history ingest requires a Snowflake connection, got ${String(inputDriver)}`);
}
return {
async executeQuery(query: string) {
@ -326,10 +323,9 @@ function historicSqlOptionsForLocalRun(project: KtxLocalProject, options: KtxCli
}
if (dialect === 'bigquery') {
const inputDriver = connection?.driver ?? 'unknown';
if (!isKtxBigQueryConnectionConfig(connection)) {
throw new Error(
`Query history ingest requires a BigQuery connection, got ${String(connection?.driver ?? 'unknown')}`,
);
throw new Error(`Query history ingest requires a BigQuery connection, got ${String(inputDriver)}`);
}
return {
...base,

3
packages/cli/src/local-scan-connectors.test.ts
View file

@ -49,7 +49,6 @@ describe('createKtxCliScanConnector', () => {
' warehouse:',
' driver: sqlite',
' path: warehouse.db',
' readonly: true',
'',
].join('\n'),
'utf-8',
@ -72,7 +71,6 @@ describe('createKtxCliScanConnector', () => {
' warehouse:',
' driver: bigquery',
' dataset_id: analytics',
' readonly: true',
' max_bytes_billed: "987654321"',
'',
].join('\n'),
@ -123,7 +121,6 @@ describe('createKtxCliScanConnector', () => {
' warehouse:',
' type: postgres',
' url: postgresql://example/db',
' readonly: true',
'',
].join('\n'),
'utf-8',

25
packages/cli/src/notion-page-picker-tree.test.ts
View file

@ -11,7 +11,6 @@ import {
selectAllVisible,
selectNone,
toggleChecked,
TRANSIENT_HINT_DURATION_MS,
visibleNodeIds,
type NotionPickerPageInput,
} from './notion-page-picker-tree.js';
@ -223,22 +222,24 @@ describe('bulk actions and reducer effects', () => {
});
});
it('blocks empty saves, updates search state, and quits without saving', () => {
it('prompts skip-empty confirmation on empty save, updates search state, and quits without saving', () => {
const state = buildInitialState({
tree: buildPickerTree(pages()),
existingRootPageIds: [],
currentCrawlMode: 'selected_roots',
});
const blockedSave = reducer(state, 'save-request', 9000);
expect(blockedSave).toEqual({
next: {
...state,
transientHint: {
text: 'Select at least one page or press q to quit',
expiresAt: 9000 + TRANSIENT_HINT_DURATION_MS,
},
},
const emptySave = reducer(state, 'save-request');
expect(emptySave).toEqual({
next: { ...state, pendingConfirm: 'skip-empty' },
effect: null,
});
expect(reducer(emptySave.next, 'save-confirm')).toEqual({
next: { ...state, pendingConfirm: null },
effect: 'quit-without-save',
});
expect(reducer(emptySave.next, 'save-cancel')).toEqual({
next: { ...state, pendingConfirm: null },
effect: null,
});
expect(
@ -262,7 +263,7 @@ describe('bulk actions and reducer effects', () => {
const withHint = {
...state,
transientHint: {
text: 'Select at least one page or press q to quit',
text: 'Select at least one page or press esc to cancel',
expiresAt: 11500,
},
};

15
packages/cli/src/notion-page-picker-tree.ts
View file

@ -22,7 +22,7 @@ export interface PickerState {
checked: Set<string>;
cursorId: string;
search: { editing: boolean; query: string };
pendingConfirm: 'mode-switch' | null;
pendingConfirm: 'mode-switch' | 'skip-empty' | null;
preLoadWarnings: string[];
transientHint: { text: string; expiresAt: number } | null;
currentCrawlMode: 'all_accessible' | 'selected_roots';
@ -61,7 +61,7 @@ interface MutableNode {
childIds: string[];
}
export const TRANSIENT_HINT_DURATION_MS = 2500;
const TRANSIENT_HINT_DURATION_MS = 2500;
const collator = new Intl.Collator('en', { sensitivity: 'base', numeric: true });
@ -444,7 +444,8 @@ export function buildInitialState(args: {
export function reducer(state: PickerState, cmd: PickerCommand, now = Date.now()): { next: PickerState; effect: PickerEffect } {
if (state.pendingConfirm) {
if (cmd === 'save-confirm') {
return { next: cloneState(state, { pendingConfirm: null }), effect: 'save' };
const effect: PickerEffect = state.pendingConfirm === 'skip-empty' ? 'quit-without-save' : 'save';
return { next: cloneState(state, { pendingConfirm: null }), effect };
}
if (cmd === 'save-cancel') {
return { next: cloneState(state, { pendingConfirm: null }), effect: null };
@ -498,19 +499,13 @@ export function reducer(state: PickerState, cmd: PickerCommand, now = Date.now()
};
case 'save-request':
if (state.checked.size === 0) {
return {
next: cloneState(state, {
transientHint: transientHint('Select at least one page or press q to quit', now),
}),
effect: null,
};
return { next: cloneState(state, { pendingConfirm: 'skip-empty' }), effect: null };
}
if (state.currentCrawlMode === 'all_accessible') {
return { next: cloneState(state, { pendingConfirm: 'mode-switch' }), effect: null };
}
return { next: state, effect: 'save' };
case 'save-confirm':
return { next: state, effect: 'save' };
case 'save-cancel':
return { next: state, effect: null };
case 'quit':

70
packages/cli/src/notion-page-picker-tui.test.tsx
View file

@ -1,7 +1,7 @@
/* @jsxImportSource react */
import { render as renderInkTest } from 'ink-testing-library';
import { act, type ReactNode } from 'react';
import { afterEach, describe, expect, it, vi } from 'vitest';
import { type ReactNode } from 'react';
import { describe, expect, it, vi } from 'vitest';
import { buildInitialState, buildPickerTree, type NotionPickerPageInput } from './notion-page-picker-tree.js';
import {
NotionPickerApp,
@ -70,13 +70,9 @@ function fakeInkInstance(): NotionPickerInkInstance {
}
function normalizeFrameWrap(frame: string | undefined): string {
return frame?.replace(/\n/g, ' ') ?? '';
return frame?.replace(/\n/g, ' ').replace(/│ /g, '').replace(/ +/g, ' ') ?? '';
}
afterEach(() => {
vi.useRealTimers();
});
describe('notionPickerCommandForInkInput', () => {
it('maps browse, search, and confirm input to reducer commands', () => {
expect(notionPickerCommandForInkInput('', { downArrow: true }, state().search, null)).toBe('cursor-down');
@ -87,9 +83,11 @@ describe('notionPickerCommandForInkInput', () => {
expect(notionPickerCommandForInkInput('/', {}, state().search, null)).toBe('search-start');
expect(notionPickerCommandForInkInput('a', {}, state().search, null)).toBe('select-all-visible');
expect(notionPickerCommandForInkInput('n', {}, state().search, null)).toBe('select-none');
expect(notionPickerCommandForInkInput('s', {}, state().search, null)).toBe('save-request');
expect(notionPickerCommandForInkInput('q', {}, state().search, null)).toBe('quit');
expect(notionPickerCommandForInkInput('', { return: true }, state().search, null)).toBe('save-request');
expect(notionPickerCommandForInkInput('', { escape: true }, state().search, null)).toBe('quit');
expect(notionPickerCommandForInkInput('c', { ctrl: true }, state().search, null)).toBe('quit');
expect(notionPickerCommandForInkInput('s', {}, state().search, null)).toBeNull();
expect(notionPickerCommandForInkInput('q', {}, state().search, null)).toBeNull();
expect(notionPickerCommandForInkInput('x', {}, { editing: true, query: '' }, null)).toEqual({
type: 'search-input',
@ -145,13 +143,16 @@ describe('NotionPickerApp', () => {
);
const frame = lastFrame() ?? '';
expect(frame).toContain('Notion pages visible to integration "Design Workspace"');
expect(frame).toContain('Select Notion pages to ingest');
expect(frame).toContain('Workspace: Design Workspace');
expect(frame).toContain('5000-page cap reached - some pages not shown');
expect(frame).toContain('1 stored root_page_ids no longer visible - they will be removed if you save');
expect(frame).toContain('▸ [ ] Engineering Docs ▸ (1)');
expect(frame).toContain(' [ ] Marketing');
expect(frame).toContain('◻ Engineering Docs ▸ (1)');
expect(frame).toContain('◻ Marketing');
expect(frame).not.toContain('Search ready: -');
expect(frame).toContain('space toggle · enter expand · / search · a all · n none · s save & exit · q quit');
expect(normalizeFrameWrap(frame)).toContain(
'Right Arrow to expand, Up/Down to move, Space to select or unselect, Slash to filter, Enter to confirm, Escape to go back, or Ctrl+C to exit.',
);
});
it('renders partial discovery warnings without stale-root save suffix', () => {
@ -199,8 +200,8 @@ describe('NotionPickerApp', () => {
);
const frame = lastFrame() ?? '';
expect(frame).toContain('▸ [×] Engineering Docs ▾');
expect(frame).toContain(' [~] Architecture');
expect(frame).toContain('◼ Engineering Docs ▾');
expect(frame).toContain(' ◼ Architecture');
});
it('supports keyboard selection, all_accessible confirmation, and save callback', async () => {
@ -220,12 +221,12 @@ describe('NotionPickerApp', () => {
stdin.write(' ');
await waitForInkInput();
expect(lastFrame()).toContain('[×] Engineering Docs');
expect(lastFrame()).toContain('◼ Engineering Docs');
stdin.write('s');
stdin.write('\r');
await waitForInkInput();
expect(normalizeFrameWrap(lastFrame())).toContain(
'Save will switch crawl_mode all_accessible -> selected_roots and limit ingest to 1 selected page. [y] confirm [esc] back',
'Switch crawl_mode from all_accessible to selected_roots? Will limit ingest to 1 selected page. Press Enter to confirm or Escape to go back.',
);
stdin.write('y');
@ -233,8 +234,7 @@ describe('NotionPickerApp', () => {
expect(onExit).toHaveBeenCalledWith({ kind: 'save', rootPageIds: [IDS.engineering] });
});
it('removes transient hints after their expiry time', async () => {
vi.useFakeTimers();
it('prompts skip-empty confirmation on empty submit and dismisses on cancel', async () => {
const onExit = vi.fn();
const { stdin, lastFrame } = renderInkTest(
<NotionPickerApp
@ -249,17 +249,25 @@ describe('NotionPickerApp', () => {
/>,
);
await act(async () => {
stdin.write('s');
await vi.advanceTimersByTimeAsync(10);
});
expect(lastFrame()).toContain('Select at least one page or press q to quit');
await act(async () => {
await vi.advanceTimersByTimeAsync(2500);
});
expect(lastFrame()).not.toContain('Select at least one page or press q to quit');
stdin.write('\r');
await waitForInkInput();
expect(normalizeFrameWrap(lastFrame())).toContain(
'Nothing selected. Skip this step? Press Enter to skip or Escape to go back.',
);
expect(onExit).not.toHaveBeenCalled();
stdin.write('n');
await waitForInkInput();
expect(lastFrame()).not.toContain('Nothing selected. Skip this step?');
expect(onExit).not.toHaveBeenCalled();
stdin.write('\r');
await waitForInkInput();
expect(lastFrame()).toContain('Nothing selected. Skip this step?');
stdin.write('\r');
await waitForInkInput();
expect(onExit).toHaveBeenCalledWith({ kind: 'quit' });
});
it('renders row-window overflow indicators when the visible list is clipped', async () => {
@ -312,7 +320,7 @@ describe('NotionPickerApp', () => {
/>,
);
stdin.write('q');
stdin.write('\u0003');
await waitForInkInput();
expect(onExit).toHaveBeenCalledWith({ kind: 'quit' });
});

102
packages/cli/src/notion-page-picker-tui.tsx
View file

@ -16,6 +16,7 @@ const COLOR_THEME = {
text: 'white',
muted: 'gray',
active: 'cyan',
selected: 'green',
warning: 'yellow',
} as const;
@ -23,6 +24,7 @@ const NO_COLOR_THEME = {
text: 'white',
muted: 'white',
active: 'white',
selected: 'white',
warning: 'white',
} as const;
@ -158,13 +160,12 @@ export function notionPickerCommandForInkInput(
if (key.downArrow) return 'cursor-down';
if (key.leftArrow) return 'cursor-left';
if (key.rightArrow) return 'cursor-right';
if (key.return) return 'expand';
if (key.return) return 'save-request';
if (input === ' ') return 'toggle-check';
if (input === '/') return 'search-start';
if (input === 'a') return 'select-all-visible';
if (input === 'n') return 'select-none';
if (input === 's') return 'save-request';
if (input === 'q' || key.escape) return 'quit';
if (key.escape) return 'quit';
return null;
}
@ -174,18 +175,27 @@ function PickerRow(props: { state: PickerState; nodeId: string; width: number; t
const focused = props.state.cursorId === node.id;
const locked = isAncestorChecked(node.id, props.state.checked, props.state.byId);
const checked = props.state.checked.has(node.id);
const glyph = locked ? '[~]' : checked ? '[×]' : '[ ]';
const children =
const isSelected = checked || locked;
const glyph = isSelected ? '◼' : '◻';
const glyphColor = locked ? props.theme.muted : checked ? props.theme.selected : props.theme.muted;
const childAffordance =
node.childIds.length > 0 ? (props.state.expanded.has(node.id) ? ' ▾' : ` ▸ (${node.childIds.length})`) : '';
const prefix = `${focused ? '▸' : ' '} ${glyph} ${' '.repeat(node.depth * 2)}`;
const color = focused ? props.theme.active : locked || node.archived ? props.theme.muted : props.theme.text;
const title = truncateText(`${node.title}${children}`, Math.max(10, props.width - prefix.length));
const indent = ' '.repeat(node.depth * 2);
const titleColor = focused ? props.theme.text : props.theme.muted;
const inverse = rowMatchesSearch(props.state, node.id);
const prefixWidth = indent.length + 2;
const title = truncateText(`${node.title}${childAffordance}`, Math.max(10, props.width - prefixWidth));
return (
<Text color={color} strikethrough={node.archived}>
{prefix}
<Text inverse={inverse}>{title}</Text>
<Text>
<Text color={glyphColor}>
{indent}
{glyph}
</Text>
<Text color={titleColor} strikethrough={node.archived}>
{' '}
<Text inverse={inverse}>{title}</Text>
</Text>
</Text>
);
}
@ -198,7 +208,7 @@ export function NotionPickerApp(props: NotionPickerAppProps): ReactNode {
const visibleIds = visibleNodeIds(state);
const selectedIndex = Math.max(0, visibleIds.indexOf(state.cursorId));
const reservedRows = state.pendingConfirm === 'mode-switch' ? 9 : 8;
const visibleRows = Math.max(5, Math.min(20, (props.terminalRows ?? 24) - reservedRows));
const visibleRows = Math.max(5, Math.min(12, (props.terminalRows ?? 24) - reservedRows));
const rows = windowItems(visibleIds, selectedIndex, visibleRows);
const hiddenAbove = rows.offset;
const hiddenBelow = Math.max(0, visibleIds.length - rows.offset - rows.items.length);
@ -254,34 +264,60 @@ export function NotionPickerApp(props: NotionPickerAppProps): ReactNode {
return (
<Box flexDirection="column">
<Text color={theme.active}>Notion pages visible to integration "{props.workspaceLabel}"</Text>
{props.cappedAtCount ? <Text color={theme.warning}>{props.cappedAtCount}-page cap reached - some pages not shown</Text> : null}
{state.preLoadWarnings.map((warning) => (
<Text key={warning} color={theme.warning}>
{staleWarningText(warning)}
</Text>
))}
{showSearch ? (
<Text>
<Text color={theme.active}>◆</Text>
<Text bold> Select Notion pages to ingest</Text>
</Text>
<Box
flexDirection="column"
borderStyle="single"
borderTop={false}
borderRight={false}
borderBottom={false}
borderColor={theme.active}
paddingLeft={1}
>
<Text color={theme.muted}>
/ {state.search.query}
{state.search.editing ? '█' : ''} ({searchMatchCount} matches)
Right Arrow to expand, Up/Down to move, Space to select or unselect, Slash to filter, Enter to confirm, Escape
to go back, or Ctrl+C to exit.
</Text>
) : null}
<Box flexDirection="column">
<Text> </Text>
<Text color={theme.muted}>Workspace: {props.workspaceLabel}</Text>
{props.cappedAtCount ? (
<Text color={theme.warning}>{props.cappedAtCount}-page cap reached - some pages not shown</Text>
) : null}
{state.preLoadWarnings.map((warning) => (
<Text key={warning} color={theme.warning}>
{staleWarningText(warning)}
</Text>
))}
{showSearch ? (
<Text>
<Text color={theme.muted}>/ </Text>
<Text>
{state.search.query}
{state.search.editing ? '█' : ''}
</Text>
<Text color={theme.muted}> ({searchMatchCount} matches)</Text>
</Text>
) : null}
{hiddenAbove > 0 ? <Text color={theme.muted}>↑ {hiddenAbove} more</Text> : null}
{rows.items.map((nodeId) => (
<PickerRow key={nodeId} state={state} nodeId={nodeId} width={width} theme={theme} />
))}
{hiddenBelow > 0 ? <Text color={theme.muted}>↓ {hiddenBelow} more</Text> : null}
{state.pendingConfirm === 'mode-switch' ? (
<Text color={theme.warning}>
Switch crawl_mode from all_accessible to selected_roots? Will limit ingest to{' '}
{selectedPageCountText(selectedCount)}. Press Enter to confirm or Escape to go back.
</Text>
) : null}
{state.pendingConfirm === 'skip-empty' ? (
<Text color={theme.warning}>Nothing selected. Skip this step? Press Enter to skip or Escape to go back.</Text>
) : null}
{state.transientHint ? <Text color={theme.warning}>{state.transientHint.text}</Text> : null}
</Box>
{state.pendingConfirm === 'mode-switch' ? (
<Text color={theme.warning}>
Save will switch crawl_mode all_accessible -&gt; selected_roots and limit ingest to{' '}
{selectedPageCountText(selectedCount)}. [y] confirm [esc] back
</Text>
) : null}
{state.transientHint ? <Text color={theme.warning}>{state.transientHint.text}</Text> : null}
<Text color={theme.muted}>space toggle · enter expand · / search · a all · n none · s save &amp; exit · q quit</Text>
<Text color={theme.active}>└</Text>
</Box>
);
}
@ -323,7 +359,7 @@ export async function renderNotionPickerTui(
exitOnCtrlC: false,
patchConsole: false,
maxFps: 30,
alternateScreen: true,
alternateScreen: false,
},
);
await instance.waitUntilExit();

2
packages/cli/src/project-dir.test.ts
View file

@ -58,7 +58,7 @@ describe('project directory defaults', () => {
argv: ['setup', '--no-input'],
spy: setup,
expected: { command: 'run', projectDir: '/tmp/ktx-env-project' },
expectedStderr: 'Project: /tmp/ktx-env-project\n',
expectedStderr: '',
},
{
argv: ['ingest', 'warehouse', '--no-input'],

7
packages/cli/src/scan.test.ts
View file

@ -861,7 +861,6 @@ describe('runKtxScan', () => {
' warehouse:',
' driver: mysql',
' url: env:MYSQL_URL',
' readonly: true',
'',
].join('\n'),
'utf-8',
@ -910,7 +909,6 @@ describe('runKtxScan', () => {
' warehouse:',
' driver: sqlite',
' path: warehouse.db',
' readonly: true',
'',
].join('\n'),
'utf-8',
@ -968,7 +966,6 @@ describe('runKtxScan', () => {
' database: analytics',
' username: reader',
' password: env:POSTGRES_PASSWORD',
' readonly: true',
'',
].join('\n'),
'utf-8',
@ -1035,7 +1032,6 @@ describe('runKtxScan', () => {
' database: analytics',
' username: reader',
' password: env:CLICKHOUSE_PASSWORD',
' readonly: true',
'',
].join('\n'),
'utf-8',
@ -1087,7 +1083,6 @@ describe('runKtxScan', () => {
' database: analytics',
' username: reader',
' schema: dbo',
' readonly: true',
'',
].join('\n'),
'utf-8',
@ -1153,7 +1148,6 @@ describe('runKtxScan', () => {
' dataset_id: analytics',
' credentials_json: env:BIGQUERY_CREDENTIALS_JSON',
' location: US',
' readonly: true',
'',
].join('\n'),
'utf-8',
@ -1222,7 +1216,6 @@ describe('runKtxScan', () => {
' database: ANALYTICS',
' schema_name: PUBLIC',
' username: reader',
' readonly: true',
'',
].join('\n'),
'utf-8',

2
packages/cli/src/setup-agents.ts
View file

@ -82,7 +82,7 @@ export function plannedKtxAgentFiles(input: {
{ kind: 'file', path: join(codexHome, 'instructions/ktx.md'), role: 'rule' as const },
];
}
throw new Error(`Global ${input.target} installation is not supported; use --project.`);
throw new Error(`Global ${input.target} installation is not supported; omit --global.`);
}
const root = resolve(input.projectDir);

414
packages/cli/src/setup-databases.test.ts
View file

@ -226,7 +226,6 @@ describe('setup databases step', () => {
' warehouse:',
' driver: postgres',
' url: env:DATABASE_URL',
' readonly: true',
'',
].join('\n'),
'utf-8',
@ -249,8 +248,9 @@ describe('setup databases step', () => {
expect(prompts.select).toHaveBeenCalledWith({
message: 'Configure PostgreSQL',
options: [
{ value: 'existing:warehouse', label: 'Use existing PostgreSQL connection: warehouse' },
{ value: 'new', label: 'Add new PostgreSQL connection' },
{ value: 'existing:warehouse', label: 'Keep existing PostgreSQL connection: warehouse' },
{ value: 'edit:warehouse', label: 'Edit PostgreSQL connection: warehouse' },
{ value: 'new', label: 'Add another PostgreSQL connection' },
{ value: 'back', label: 'Back' },
],
});
@ -290,7 +290,6 @@ describe('setup databases step', () => {
expect(config.connections['postgres-warehouse']).toEqual({
driver: 'postgres',
url: 'env:DATABASE_URL',
readonly: true,
context: { depth: 'fast' },
});
});
@ -560,7 +559,6 @@ describe('setup databases step', () => {
' warehouse:',
' driver: postgres',
' url: env:DATABASE_URL',
' readonly: true',
'setup:',
' database_connection_ids:',
' - warehouse',
@ -591,6 +589,7 @@ describe('setup databases step', () => {
message: 'Databases already configured: warehouse\nWhat would you like to do?',
options: [
{ value: 'continue', label: 'Continue to context sources' },
{ value: 'edit', label: 'Edit an existing database' },
{ value: 'add', label: 'Add another database' },
],
});
@ -607,7 +606,6 @@ describe('setup databases step', () => {
' warehouse:',
' driver: postgres',
' url: env:DATABASE_URL',
' readonly: true',
'setup:',
' database_connection_ids:',
' - warehouse',
@ -642,10 +640,15 @@ describe('setup databases step', () => {
connectionIds: ['warehouse', 'mysql-warehouse'],
});
expect(prompts.multiselect).toHaveBeenCalledTimes(1);
expect(prompts.multiselect).toHaveBeenCalledWith(expect.objectContaining({
initialValues: ['postgres'],
required: true,
}));
expect(prompts.select).toHaveBeenCalledWith({
message: 'Databases already configured: warehouse\nWhat would you like to do?',
options: [
{ value: 'continue', label: 'Continue to context sources' },
{ value: 'edit', label: 'Edit an existing database' },
{ value: 'add', label: 'Add another database' },
],
});
@ -683,10 +686,15 @@ describe('setup databases step', () => {
connectionIds: ['postgres-warehouse', 'mysql-warehouse'],
});
expect(prompts.multiselect).toHaveBeenCalledTimes(2);
expect(prompts.multiselect).toHaveBeenNthCalledWith(2, expect.objectContaining({
initialValues: ['postgres'],
required: true,
}));
expect(prompts.select).toHaveBeenCalledWith({
message: 'Databases already configured: postgres-warehouse\nWhat would you like to do?',
options: [
{ value: 'continue', label: 'Continue to context sources' },
{ value: 'edit', label: 'Edit an existing database' },
{ value: 'add', label: 'Add another database' },
],
});
@ -722,11 +730,16 @@ describe('setup databases step', () => {
connectionIds: ['postgres-warehouse'],
});
expect(prompts.multiselect).toHaveBeenCalledTimes(2);
expect(prompts.multiselect).toHaveBeenNthCalledWith(2, expect.objectContaining({
initialValues: ['postgres'],
required: true,
}));
expect(io.stdout()).not.toContain('KTX cannot work without at least one database');
expect(prompts.select).toHaveBeenNthCalledWith(3, {
message: 'Databases already configured: postgres-warehouse\nWhat would you like to do?',
options: [
{ value: 'continue', label: 'Continue to context sources' },
{ value: 'edit', label: 'Edit an existing database' },
{ value: 'add', label: 'Add another database' },
],
});
@ -741,7 +754,6 @@ describe('setup databases step', () => {
' warehouse:',
' driver: postgres',
' url: env:DATABASE_URL',
' readonly: true',
'setup:',
' database_connection_ids:',
' - warehouse',
@ -763,16 +775,394 @@ describe('setup databases step', () => {
);
expect(result).toEqual({ status: 'ready', projectDir: tempDir, connectionIds: ['warehouse'] });
expect(prompts.multiselect).toHaveBeenCalledWith(expect.objectContaining({
initialValues: ['postgres'],
required: true,
}));
expect(io.stdout()).not.toContain('KTX cannot work without at least one database');
expect(prompts.select).toHaveBeenNthCalledWith(2, {
message: 'Databases already configured: warehouse\nWhat would you like to do?',
options: [
{ value: 'continue', label: 'Continue to context sources' },
{ value: 'edit', label: 'Edit an existing database' },
{ value: 'add', label: 'Add another database' },
],
});
});
it('returns from database edit selection back to the configured source menu', async () => {
await writeFile(
join(tempDir, 'ktx.yaml'),
[
'project: warehouse',
'connections:',
' warehouse:',
' driver: postgres',
' url: env:DATABASE_URL',
'setup:',
' database_connection_ids:',
' - warehouse',
'',
].join('\n'),
'utf-8',
);
await writeKtxSetupState(tempDir, { completed_steps: ['databases'] });
const prompts = makePromptAdapter({
selectValues: ['edit', 'back', 'continue'],
});
const testConnection = vi.fn(async () => 0);
const scanConnection = vi.fn(async () => 0);
const result = await runKtxSetupDatabasesStep(
{ projectDir: tempDir, inputMode: 'auto', skipDatabases: false, databaseSchemas: [] },
makeIo().io,
{ prompts, testConnection, scanConnection },
);
expect(result).toEqual({ status: 'ready', projectDir: tempDir, connectionIds: ['warehouse'] });
expect(prompts.select).toHaveBeenNthCalledWith(2, {
message: 'Database to edit',
options: [
{ value: 'warehouse', label: 'warehouse (PostgreSQL)' },
{ value: 'back', label: 'Back' },
],
});
expect(prompts.select).toHaveBeenNthCalledWith(3, {
message: 'Databases already configured: warehouse\nWhat would you like to do?',
options: [
{ value: 'continue', label: 'Continue to context sources' },
{ value: 'edit', label: 'Edit an existing database' },
{ value: 'add', label: 'Add another database' },
],
});
expect(testConnection).not.toHaveBeenCalled();
expect(scanConnection).not.toHaveBeenCalled();
});
it('reruns table selection after editing schema scope so stale enabled tables are removed', async () => {
await writeFile(
join(tempDir, 'ktx.yaml'),
[
'project: warehouse',
'connections:',
' warehouse:',
' driver: postgres',
' url: env:DATABASE_URL',
' schemas:',
' - public',
' enabled_tables:',
' - public.orders',
'setup:',
' database_connection_ids:',
' - warehouse',
'',
].join('\n'),
'utf-8',
);
await writeKtxSetupState(tempDir, { completed_steps: ['databases'] });
const prompts = makePromptAdapter({
textValues: ['env:DATABASE_URL'],
multiselectValues: [['analytics']],
});
let primaryMenuCount = 0;
vi.mocked(prompts.select).mockImplementation(async (options) => {
if (options.message === 'Databases already configured: warehouse\nWhat would you like to do?') {
primaryMenuCount += 1;
return primaryMenuCount === 1 ? 'edit' : 'continue';
}
if (options.message === 'Database to edit') return 'warehouse';
if (options.message === 'How do you want to connect to PostgreSQL?') return 'url';
if (options.message.startsWith('Enable query-history ingest')) return 'no';
return 'back';
});
const testConnection = vi.fn(async () => 0);
const scanConnection = vi.fn(async () => 0);
const listSchemas = vi.fn(async () => ['analytics', 'public']);
const listTables = vi.fn(async () => [{ schema: 'analytics', name: 'customers', kind: 'table' as const }]);
const result = await runKtxSetupDatabasesStep(
{ projectDir: tempDir, inputMode: 'auto', skipDatabases: false, databaseSchemas: [] },
makeIo().io,
{ prompts, testConnection, scanConnection, listSchemas, listTables },
);
expect(result).toEqual({ status: 'ready', projectDir: tempDir, connectionIds: ['warehouse'] });
expect(prompts.text).toHaveBeenCalledWith({
message: textInputPrompt('PostgreSQL connection URL'),
placeholder: 'env:DATABASE_URL',
initialValue: 'env:DATABASE_URL',
});
expect(listTables).toHaveBeenCalledWith(tempDir, 'warehouse');
expect(testConnection).toHaveBeenCalledWith(tempDir, 'warehouse', expect.anything());
expect(scanConnection).toHaveBeenCalledWith(tempDir, 'warehouse', expect.anything());
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
expect(config.connections.warehouse).toMatchObject({
schemas: ['analytics'],
enabled_tables: ['analytics.customers'],
});
});
it('preselects existing schema and table choices when editing a database', async () => {
await writeFile(
join(tempDir, 'ktx.yaml'),
[
'project: warehouse',
'connections:',
' warehouse:',
' driver: postgres',
' url: env:DATABASE_URL',
' schemas:',
' - public',
' enabled_tables:',
' - public.customers',
' - public.orders',
'setup:',
' database_connection_ids:',
' - warehouse',
'',
].join('\n'),
'utf-8',
);
await writeKtxSetupState(tempDir, { completed_steps: ['databases'] });
const prompts = makePromptAdapter({
textValues: ['env:DATABASE_URL'],
multiselectValues: [['public'], ['public.customers', 'public.orders']],
});
let primaryMenuCount = 0;
vi.mocked(prompts.select).mockImplementation(async (options) => {
if (options.message === 'Databases already configured: warehouse\nWhat would you like to do?') {
primaryMenuCount += 1;
return primaryMenuCount === 1 ? 'edit' : 'continue';
}
if (options.message === 'Database to edit') return 'warehouse';
if (options.message === 'How do you want to connect to PostgreSQL?') return 'url';
if (options.message.startsWith('Enable query-history ingest')) return 'no';
if (options.message.startsWith('Tables found in selected schemas')) return 'customize';
return 'back';
});
const listSchemas = vi.fn(async () => ['orbit_analytics', 'orbit_raw', 'public']);
const listTables = vi.fn(async () => [
{ schema: 'public', name: 'customers', kind: 'table' as const },
{ schema: 'public', name: 'orders', kind: 'table' as const },
{ schema: 'public', name: 'products', kind: 'table' as const },
]);
const result = await runKtxSetupDatabasesStep(
{ projectDir: tempDir, inputMode: 'auto', skipDatabases: false, databaseSchemas: [] },
makeIo().io,
{
prompts,
testConnection: vi.fn(async () => 0),
scanConnection: vi.fn(async () => 0),
listSchemas,
listTables,
},
);
expect(result).toEqual({ status: 'ready', projectDir: tempDir, connectionIds: ['warehouse'] });
expect(prompts.multiselect).toHaveBeenNthCalledWith(1, {
message: expect.stringContaining('PostgreSQL schemas to include'),
options: [
{ value: 'orbit_analytics', label: 'orbit_analytics' },
{ value: 'orbit_raw', label: 'orbit_raw' },
{ value: 'public', label: 'public' },
],
initialValues: ['public'],
required: true,
});
expect(prompts.multiselect).toHaveBeenNthCalledWith(2, {
message: expect.stringContaining('Tables to enable for warehouse'),
options: [
{ value: 'public.customers', label: 'public.customers' },
{ value: 'public.orders', label: 'public.orders' },
{ value: 'public.products', label: 'public.products' },
],
initialValues: ['public.customers', 'public.orders'],
required: true,
});
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
expect(config.connections.warehouse).toMatchObject({
schemas: ['public'],
enabled_tables: ['public.customers', 'public.orders'],
});
});
it('returns to the configured primary menu when backing out of schema review during edit', async () => {
await writeFile(
join(tempDir, 'ktx.yaml'),
[
'project: warehouse',
'connections:',
' warehouse:',
' driver: postgres',
' url: env:DATABASE_URL',
' schemas:',
' - public',
' enabled_tables:',
' - public.orders',
'setup:',
' database_connection_ids:',
' - warehouse',
'',
].join('\n'),
'utf-8',
);
await writeKtxSetupState(tempDir, { completed_steps: ['databases'] });
const prompts = makePromptAdapter({
textValues: ['env:DATABASE_URL'],
multiselectValues: [['back']],
});
let primaryMenuCount = 0;
vi.mocked(prompts.select).mockImplementation(async (options) => {
if (options.message === 'Databases already configured: warehouse\nWhat would you like to do?') {
primaryMenuCount += 1;
return primaryMenuCount === 1 ? 'edit' : 'continue';
}
if (options.message === 'Database to edit') return 'warehouse';
if (options.message === 'How do you want to connect to PostgreSQL?') return 'url';
if (options.message.startsWith('Enable query-history ingest')) return 'no';
return 'back';
});
const testConnection = vi.fn(async () => 0);
const scanConnection = vi.fn(async () => 0);
const listSchemas = vi.fn(async () => ['analytics', 'public']);
const listTables = vi.fn(async () => [{ schema: 'analytics', name: 'customers', kind: 'table' as const }]);
const result = await runKtxSetupDatabasesStep(
{ projectDir: tempDir, inputMode: 'auto', skipDatabases: false, databaseSchemas: [] },
makeIo().io,
{ prompts, testConnection, scanConnection, listSchemas, listTables },
);
expect(result).toEqual({ status: 'ready', projectDir: tempDir, connectionIds: ['warehouse'] });
expect(primaryMenuCount).toBe(2);
expect(testConnection).toHaveBeenCalledWith(tempDir, 'warehouse', expect.anything());
expect(scanConnection).not.toHaveBeenCalled();
expect(listTables).not.toHaveBeenCalled();
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
expect(config.connections.warehouse).toMatchObject({
url: 'env:DATABASE_URL',
schemas: ['public'],
enabled_tables: ['public.orders'],
});
});
it('returns to the configured primary menu when backing out of table review during edit', async () => {
await writeFile(
join(tempDir, 'ktx.yaml'),
[
'project: warehouse',
'connections:',
' warehouse:',
' driver: postgres',
' url: env:DATABASE_URL',
' schemas:',
' - public',
' enabled_tables:',
' - public.orders',
'setup:',
' database_connection_ids:',
' - warehouse',
'',
].join('\n'),
'utf-8',
);
await writeKtxSetupState(tempDir, { completed_steps: ['databases'] });
const prompts = makePromptAdapter({ textValues: ['env:DATABASE_URL'] });
let primaryMenuCount = 0;
vi.mocked(prompts.select).mockImplementation(async (options) => {
if (options.message === 'Databases already configured: warehouse\nWhat would you like to do?') {
primaryMenuCount += 1;
return primaryMenuCount === 1 ? 'edit' : 'continue';
}
if (options.message === 'Database to edit') return 'warehouse';
if (options.message === 'How do you want to connect to PostgreSQL?') return 'url';
if (options.message.startsWith('Enable query-history ingest')) return 'no';
if (options.message.startsWith('Tables found in selected schemas')) return 'back';
return 'back';
});
const testConnection = vi.fn(async () => 0);
const scanConnection = vi.fn(async () => 0);
const listSchemas = vi.fn(async () => ['public']);
const listTables = vi.fn(async () => [
{ schema: 'public', name: 'customers', kind: 'table' as const },
{ schema: 'public', name: 'orders', kind: 'table' as const },
]);
const result = await runKtxSetupDatabasesStep(
{ projectDir: tempDir, inputMode: 'auto', skipDatabases: false, databaseSchemas: [] },
makeIo().io,
{ prompts, testConnection, scanConnection, listSchemas, listTables },
);
expect(result).toEqual({ status: 'ready', projectDir: tempDir, connectionIds: ['warehouse'] });
expect(primaryMenuCount).toBe(2);
expect(listTables).toHaveBeenCalledWith(tempDir, 'warehouse');
expect(scanConnection).not.toHaveBeenCalled();
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
expect(config.connections.warehouse).toMatchObject({
url: 'env:DATABASE_URL',
schemas: ['public'],
enabled_tables: ['public.orders'],
});
});
it('restores an existing database edit when the follow-up scan fails', async () => {
await writeFile(
join(tempDir, 'ktx.yaml'),
[
'project: warehouse',
'connections:',
' warehouse:',
' driver: postgres',
' url: env:DATABASE_URL',
' schemas:',
' - public',
' enabled_tables:',
' - public.orders',
'setup:',
' database_connection_ids:',
' - warehouse',
'',
].join('\n'),
'utf-8',
);
await writeKtxSetupState(tempDir, { completed_steps: ['databases'] });
const prompts = makePromptAdapter({
textValues: ['env:DATABASE_URL'],
multiselectValues: [['public']],
});
vi.mocked(prompts.select).mockImplementation(async (options) => {
if (options.message === 'Databases already configured: warehouse\nWhat would you like to do?') return 'edit';
if (options.message === 'Database to edit') return 'warehouse';
if (options.message === 'How do you want to connect to PostgreSQL?') return 'url';
if (options.message.startsWith('Enable query-history ingest')) return 'no';
if (options.message.startsWith('Tables found in selected schemas')) return 'all';
return 'back';
});
const listTables = vi.fn(async () => [
{ schema: 'public', name: 'customers', kind: 'table' as const },
{ schema: 'public', name: 'orders', kind: 'table' as const },
]);
const result = await runKtxSetupDatabasesStep(
{ projectDir: tempDir, inputMode: 'auto', skipDatabases: false, databaseSchemas: [] },
makeIo().io,
{
prompts,
testConnection: vi.fn(async () => 0),
scanConnection: vi.fn(async () => 1),
listTables,
},
);
expect(result).toEqual({ status: 'failed', projectDir: tempDir });
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
expect(config.connections.warehouse).toMatchObject({
enabled_tables: ['public.orders'],
});
});
it('lets Escape from connection fields return to connection method selection', async () => {
const prompts = makePromptAdapter({
selectValues: ['fields', 'url'],
@ -887,7 +1277,6 @@ describe('setup databases step', () => {
port: 5432,
database: 'analytics',
username: 'readonly',
readonly: true,
});
expect(connection.password).toMatch(/^file:/);
const secretPath = join(tempDir, '.ktx/secrets/postgres-warehouse-password');
@ -941,7 +1330,7 @@ describe('setup databases step', () => {
return 0;
});
const scanConnection = vi.fn(async (_projectDir: string, _connectionId: string, commandIo: KtxCliIo) => {
commandIo.stdout.write('Scanning postgres-warehouse for context. Large primary sources can take a while.\n');
commandIo.stdout.write('Scanning postgres-warehouse for context. Large databases can take a while.\n');
commandIo.stdout.write('[5%] Preparing scan\n');
commandIo.stdout.write('[15%] Inspecting database schema\n');
commandIo.stdout.write('[55%] Semantic layer comparison found 2 changes across 2 tables\n');
@ -1038,7 +1427,6 @@ describe('setup databases step', () => {
expect(config.connections['postgres-warehouse']).toMatchObject({
driver: 'postgres',
url: 'env:DATABASE_URL',
readonly: true,
});
});
@ -1158,7 +1546,6 @@ describe('setup databases step', () => {
url: 'env:DATABASE_URL',
schemas: ['public'],
context: { queryHistory: { enabled: false }, depth: 'fast' },
readonly: true,
});
expect(config.setup).toEqual({
database_connection_ids: ['warehouse'],
@ -1197,7 +1584,6 @@ describe('setup databases step', () => {
expect(config.connections.warehouse).toEqual({
driver: 'sqlite',
path: './warehouse.sqlite',
readonly: true,
context: { depth: 'fast' },
});
expect(config.setup).toEqual({
@ -1215,7 +1601,6 @@ describe('setup databases step', () => {
' warehouse:',
' driver: postgres',
' url: env:DATABASE_URL',
' readonly: true',
' analytics:',
' driver: snowflake',
' authMethod: password',
@ -1225,7 +1610,6 @@ describe('setup databases step', () => {
' schema_name: PUBLIC',
' username: reader',
' password: env:SNOWFLAKE_PASSWORD',
' readonly: true',
'',
].join('\n'),
'utf-8',
@ -1591,7 +1975,6 @@ describe('setup databases step', () => {
' driver: bigquery',
' dataset_id: analytics',
' credentials_json: env:BIGQUERY_CREDENTIALS_JSON',
' readonly: true',
'',
].join('\n'),
'utf-8',
@ -1646,7 +2029,6 @@ describe('setup databases step', () => {
' warehouse:',
' driver: postgres',
' url: env:DATABASE_URL',
' readonly: true',
'',
].join('\n'),
'utf-8',

498
packages/cli/src/setup-databases.ts
View file

@ -177,6 +177,7 @@ const SCOPE_DISCOVERY_SPECS: Partial<Record<KtxSetupDatabaseDriver, ScopeDiscove
};
type UrlDriverType = Extract<KtxSetupDatabaseDriver, 'postgres' | 'mysql' | 'clickhouse' | 'sqlserver'>;
type ConnectionSetupStatus = 'ready' | 'back' | 'failed';
const DRIVER_CONNECTION_DEFAULTS: Record<UrlDriverType, { port: string }> = {
postgres: { port: '5432' },
@ -234,6 +235,16 @@ function assertSafeDatabaseConnectionId(connectionId: string): void {
}
}
function stringConfigField(connection: KtxProjectConnectionConfig | undefined, field: string): string | undefined {
const value = connection?.[field];
return typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined;
}
function numberConfigField(connection: KtxProjectConnectionConfig | undefined, field: string): number | undefined {
const value = connection?.[field];
return typeof value === 'number' && Number.isFinite(value) ? value : undefined;
}
function historicSqlConfigRecord(connection: KtxProjectConnectionConfig | undefined): Record<string, unknown> | null {
const historicSql = connection?.historicSql;
return historicSql && typeof historicSql === 'object' && !Array.isArray(historicSql)
@ -503,6 +514,18 @@ function configuredPrimaryConnectionIds(
.sort((left, right) => left.localeCompare(right));
}
function configuredPrimaryDrivers(
connections: Record<string, KtxProjectConnectionConfig>,
connectionIds: string[],
): KtxSetupDatabaseDriver[] {
const configured = new Set(
connectionIds
.map((connectionId) => normalizeDriver(connections[connectionId]?.driver))
.filter((driver): driver is KtxSetupDatabaseDriver => driver !== null),
);
return DRIVER_OPTIONS.map((option) => option.value).filter((driver) => configured.has(driver));
}
function configuredPrimarySourcesPrompt(connectionIds: string[]): {
message: string;
options: Array<{ value: string; label: string }>;
@ -511,6 +534,7 @@ function configuredPrimarySourcesPrompt(connectionIds: string[]): {
message: `Databases already configured: ${connectionIds.join(', ')}\nWhat would you like to do?`,
options: [
{ value: 'continue', label: 'Continue to context sources' },
{ value: 'edit', label: 'Edit an existing database' },
{ value: 'add', label: 'Add another database' },
],
};
@ -601,23 +625,40 @@ async function buildFieldsConnectionConfig(input: {
connectionId: string;
args: KtxSetupDatabasesArgs;
prompts: KtxSetupDatabasesPromptAdapter;
existingConnection?: KtxProjectConnectionConfig;
}): Promise<KtxProjectConnectionConfig | null | 'back'> {
const label = driverLabel(input.driver);
const defaults = DRIVER_CONNECTION_DEFAULTS[input.driver];
const host = await promptText(input.prompts, `${label} host`, 'localhost');
const host = await promptText(
input.prompts,
`${label} host`,
stringConfigField(input.existingConnection, 'host') ?? 'localhost',
);
if (host === undefined) return 'back';
if (!host) return null;
const portStr = await promptText(input.prompts, `${label} port`, defaults.port);
const portStr = await promptText(
input.prompts,
`${label} port`,
String(numberConfigField(input.existingConnection, 'port') ?? defaults.port),
);
if (portStr === undefined) return 'back';
const port = Number(portStr || defaults.port);
const database = await promptText(input.prompts, `${label} database name`);
const database = await promptText(
input.prompts,
`${label} database name`,
stringConfigField(input.existingConnection, 'database'),
);
if (database === undefined) return 'back';
if (!database) return null;
const username = await promptText(input.prompts, `${label} username`);
const username = await promptText(
input.prompts,
`${label} username`,
stringConfigField(input.existingConnection, 'username'),
);
if (username === undefined) return 'back';
if (!username) return null;
@ -632,6 +673,7 @@ async function buildFieldsConnectionConfig(input: {
});
if (credentialResult === 'back') return 'back';
if (credentialResult) passwordRef = credentialResult;
if (!credentialResult) passwordRef = stringConfigField(input.existingConnection, 'password');
}
return {
@ -642,7 +684,6 @@ async function buildFieldsConnectionConfig(input: {
username,
...(passwordRef ? { password: passwordRef } : {}),
...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}),
readonly: true,
};
}
@ -651,9 +692,14 @@ async function buildPastedUrlConnectionConfig(input: {
connectionId: string;
args: KtxSetupDatabasesArgs;
prompts: KtxSetupDatabasesPromptAdapter;
existingConnection?: KtxProjectConnectionConfig;
}): Promise<KtxProjectConnectionConfig | null | 'back'> {
const label = driverLabel(input.driver);
const rawUrl = await promptText(input.prompts, `${label} connection URL`);
const rawUrl = await promptText(
input.prompts,
`${label} connection URL`,
stringConfigField(input.existingConnection, 'url'),
);
if (rawUrl === undefined) return 'back';
if (!rawUrl) return null;
@ -664,7 +710,6 @@ async function buildPastedUrlConnectionConfig(input: {
driver: input.driver,
url,
...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}),
readonly: true,
};
}
@ -678,7 +723,6 @@ async function buildPastedUrlConnectionConfig(input: {
driver: input.driver,
url: ref,
...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}),
readonly: true,
};
}
@ -686,7 +730,6 @@ async function buildPastedUrlConnectionConfig(input: {
driver: input.driver,
url,
...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}),
readonly: true,
};
}
@ -695,6 +738,7 @@ async function buildUrlConnectionConfig(input: {
connectionId: string;
args: KtxSetupDatabasesArgs;
prompts: KtxSetupDatabasesPromptAdapter;
existingConnection?: KtxProjectConnectionConfig;
}): Promise<KtxProjectConnectionConfig | null | 'back'> {
if (input.args.inputMode === 'disabled' && !input.args.databaseUrl) return null;
@ -710,14 +754,12 @@ async function buildUrlConnectionConfig(input: {
driver: input.driver,
url: ref,
...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}),
readonly: true,
};
}
return {
driver: input.driver,
url,
...(input.args.databaseSchemas.length > 0 ? { schemas: input.args.databaseSchemas } : {}),
readonly: true,
};
}
@ -744,6 +786,7 @@ async function buildConnectionConfig(input: {
connectionId: string;
args: KtxSetupDatabasesArgs;
prompts: KtxSetupDatabasesPromptAdapter;
existingConnection?: KtxProjectConnectionConfig;
}): Promise<KtxProjectConnectionConfig | null | 'back'> {
const { driver, args, prompts } = input;
if (driver === 'sqlite') {
@ -753,22 +796,37 @@ async function buildConnectionConfig(input: {
(await promptText(
prompts,
'SQLite database file\nEnter a relative or absolute path, for example ./warehouse.sqlite.',
stringConfigField(input.existingConnection, 'path'),
));
if (path === undefined) return 'back';
return path ? { driver: 'sqlite', path, readonly: true } : null;
return path ? { driver: 'sqlite', path } : null;
}
if (driver === 'postgres' || driver === 'mysql' || driver === 'clickhouse' || driver === 'sqlserver') {
return await buildUrlConnectionConfig({ driver, connectionId: input.connectionId, args, prompts });
return await buildUrlConnectionConfig({
driver,
connectionId: input.connectionId,
args,
prompts,
existingConnection: input.existingConnection,
});
}
if (driver === 'bigquery') {
const datasetId = await promptText(prompts, 'BigQuery dataset\nFor example analytics.');
const datasetId = await promptText(
prompts,
'BigQuery dataset\nFor example analytics.',
stringConfigField(input.existingConnection, 'dataset_id'),
);
if (datasetId === undefined) return 'back';
const credentialsPath = await promptText(prompts, 'Path to service account JSON file');
const credentialsPath = await promptText(
prompts,
'Path to service account JSON file',
stringConfigField(input.existingConnection, 'credentials_json'),
);
if (credentialsPath === undefined) return 'back';
const location = await promptText(
prompts,
'BigQuery location\nPress Enter for US, or enter a location like EU.',
'US',
stringConfigField(input.existingConnection, 'location') ?? 'US',
);
if (location === undefined) return 'back';
if (!datasetId || !credentialsPath) return null;
@ -777,23 +835,38 @@ async function buildConnectionConfig(input: {
dataset_id: datasetId,
credentials_json: normalizeFileReference(credentialsPath),
...(location ? { location } : {}),
readonly: true,
};
}
if (driver === 'snowflake') {
const account = await promptText(prompts, 'Snowflake account identifier');
const account = await promptText(
prompts,
'Snowflake account identifier',
stringConfigField(input.existingConnection, 'account'),
);
if (account === undefined) return 'back';
const warehouse = await promptText(prompts, 'Snowflake warehouse\nFor example ANALYTICS_WH.');
const warehouse = await promptText(
prompts,
'Snowflake warehouse\nFor example ANALYTICS_WH.',
stringConfigField(input.existingConnection, 'warehouse'),
);
if (warehouse === undefined) return 'back';
const database = await promptText(prompts, 'Snowflake database name');
const database = await promptText(
prompts,
'Snowflake database name',
stringConfigField(input.existingConnection, 'database'),
);
if (database === undefined) return 'back';
const schemaName = await promptText(
prompts,
'Snowflake schema\nPress Enter for PUBLIC, or enter a schema name.',
'PUBLIC',
stringConfigField(input.existingConnection, 'schema_name') ?? 'PUBLIC',
);
if (schemaName === undefined) return 'back';
const username = await promptText(prompts, 'Snowflake username');
const username = await promptText(
prompts,
'Snowflake username',
stringConfigField(input.existingConnection, 'username'),
);
if (username === undefined) return 'back';
const passwordRef = await promptCredential({
prompts,
@ -803,9 +876,14 @@ async function buildConnectionConfig(input: {
secretName: 'password', // pragma: allowlist secret
});
if (passwordRef === 'back') return 'back'; // pragma: allowlist secret
const role = await promptText(prompts, 'Snowflake role (optional)\nPress Enter to skip.');
const role = await promptText(
prompts,
'Snowflake role (optional)\nPress Enter to skip.',
stringConfigField(input.existingConnection, 'role'),
);
if (role === undefined) return 'back';
if (!account || !warehouse || !database || !schemaName || !username || !passwordRef) return null;
const resolvedPasswordRef = passwordRef ?? stringConfigField(input.existingConnection, 'password');
if (!account || !warehouse || !database || !schemaName || !username || !resolvedPasswordRef) return null;
return {
driver: 'snowflake',
authMethod: 'password',
@ -814,9 +892,8 @@ async function buildConnectionConfig(input: {
database,
schema_name: schemaName,
username,
password: passwordRef,
password: resolvedPasswordRef,
...(role ? { role } : {}),
readonly: true,
};
}
throw new Error(`Unsupported database driver: ${driver}`);
@ -1134,6 +1211,59 @@ async function writeConnectionConfig(input: {
}
}
async function createConnectionConfigRollback(projectDir: string, connectionId: string): Promise<() => Promise<void>> {
const project = await loadKtxProject({ projectDir });
const previousConnection = project.config.connections[connectionId];
const hadPreviousConnection = previousConnection !== undefined;
return async () => {
const latest = await loadKtxProject({ projectDir });
const connections = { ...latest.config.connections };
if (hadPreviousConnection) {
connections[connectionId] = previousConnection;
} else {
delete connections[connectionId];
}
await writeFile(
latest.configPath,
serializeKtxProjectConfig({
...latest.config,
connections,
}),
'utf-8',
);
};
}
function withExistingPrimaryEditPromptDefaults(input: {
previous: KtxProjectConnectionConfig;
next: KtxProjectConnectionConfig;
driver: KtxSetupDatabaseDriver;
}): KtxProjectConnectionConfig {
const merged: KtxProjectConnectionConfig = { ...input.next };
const spec = SCOPE_DISCOVERY_SPECS[input.driver];
if (spec) {
const nextArray = input.next[spec.configArrayField];
const previousArray = input.previous[spec.configArrayField];
if (
!(Array.isArray(nextArray) && nextArray.length > 0) &&
Array.isArray(previousArray) &&
previousArray.length > 0
) {
delete merged[spec.configSingleField];
merged[spec.configArrayField] = previousArray;
} else if (!Object.hasOwn(input.next, spec.configArrayField) && !Object.hasOwn(input.next, spec.configSingleField)) {
const previousSingle = input.previous[spec.configSingleField];
if (typeof previousSingle === 'string' && previousSingle.trim().length > 0) {
merged[spec.configSingleField] = previousSingle;
}
}
}
if (!Object.hasOwn(input.next, 'enabled_tables') && Array.isArray(input.previous.enabled_tables)) {
merged.enabled_tables = input.previous.enabled_tables;
}
return merged;
}
function configuredScopeValues(
connection: KtxProjectConnectionConfig | undefined,
spec: ScopeDiscoverySpec,
@ -1194,18 +1324,19 @@ async function maybeConfigureSchemaScope(input: {
prompts: KtxSetupDatabasesPromptAdapter;
deps: KtxSetupDatabasesDeps;
io: KtxCliIo;
}): Promise<boolean> {
forcePrompt?: boolean;
}): Promise<ConnectionSetupStatus> {
const project = await loadKtxProject({ projectDir: input.projectDir });
const connection = project.config.connections[input.connectionId];
const driver = normalizeDriver(connection?.driver);
if (!driver) return true;
if (!driver) return 'ready';
const spec = SCOPE_DISCOVERY_SPECS[driver];
if (!spec) return true;
if (!spec) return 'ready';
const arrayVal = connection?.[spec.configArrayField];
if (Array.isArray(arrayVal) && arrayVal.length > 0) {
return true;
if (Array.isArray(arrayVal) && arrayVal.length > 0 && input.forcePrompt !== true) {
return 'ready';
}
if (input.args.databaseSchemas.length > 0) {
@ -1215,7 +1346,7 @@ async function maybeConfigureSchemaScope(input: {
values: input.args.databaseSchemas,
spec,
});
return true;
return 'ready';
}
writeSetupSection(input.io, `Discovering ${spec.promptLabel.toLowerCase()}`, [
@ -1228,14 +1359,18 @@ async function maybeConfigureSchemaScope(input: {
await (input.deps.listSchemas ?? defaultListSchemas)(input.projectDir, input.connectionId),
);
} catch (error) {
const detail = error instanceof Error ? error.message : String(error);
input.io.stderr.write(
`Could not discover ${spec.promptLabel.toLowerCase()} for ${input.connectionId}; continuing with existing ${spec.noun} scope. ` +
`Pass --database-schema to set it explicitly. ${error instanceof Error ? error.message : String(error)}\n`,
input.forcePrompt === true
? `Could not discover ${spec.promptLabel.toLowerCase()} for ${input.connectionId}; edit was not saved. ` +
`Pass --database-schema to set it explicitly. ${detail}\n`
: `Could not discover ${spec.promptLabel.toLowerCase()} for ${input.connectionId}; continuing with existing ${spec.noun} scope. ` +
`Pass --database-schema to set it explicitly. ${detail}\n`,
);
return true;
return input.forcePrompt === true ? 'failed' : 'ready';
}
if (discovered.length === 0) {
return true;
return 'ready';
}
let selected: string[];
@ -1255,7 +1390,7 @@ async function maybeConfigureSchemaScope(input: {
required: true,
});
if (choices.includes('back')) {
return false;
return 'back';
}
selected = choices.length > 0 ? choices : initialValues;
}
@ -1270,7 +1405,7 @@ async function maybeConfigureSchemaScope(input: {
writeSetupSection(input.io, `${capitalNounPlural} saved for ${input.connectionId}`, [
`✓ ${selected.join(', ')}`,
]);
return true;
return 'ready';
}
async function maybeConfigureTableScope(input: {
@ -1280,19 +1415,20 @@ async function maybeConfigureTableScope(input: {
prompts: KtxSetupDatabasesPromptAdapter;
io: KtxCliIo;
deps: KtxSetupDatabasesDeps;
}): Promise<boolean> {
forcePrompt?: boolean;
}): Promise<ConnectionSetupStatus> {
const project = await loadKtxProject({ projectDir: input.projectDir });
const connection = project.config.connections[input.connectionId];
const driver = normalizeDriver(connection?.driver);
if (!driver || driver === 'sqlite') return true;
if (!driver || driver === 'sqlite') return 'ready';
const existingTables = connection?.enabled_tables;
if (Array.isArray(existingTables) && existingTables.length > 0) {
return true;
if (Array.isArray(existingTables) && existingTables.length > 0 && input.forcePrompt !== true) {
return 'ready';
}
if (input.args.inputMode === 'disabled') {
return true;
return 'ready';
}
writeSetupSection(input.io, 'Discovering tables', [
@ -1306,15 +1442,20 @@ async function maybeConfigureTableScope(input: {
input.connectionId,
);
} catch (error) {
const detail = error instanceof Error ? error.message : String(error);
input.io.stderr.write(
`Could not discover tables for ${input.connectionId}; continuing without table filter. ` +
`${error instanceof Error ? error.message : String(error)}\n`,
input.forcePrompt === true
? `Could not discover tables for ${input.connectionId}; edit was not saved. ${detail}\n`
: `Could not discover tables for ${input.connectionId}; continuing without table filter. ${detail}\n`,
);
return true;
return input.forcePrompt === true ? 'failed' : 'ready';
}
if (discovered.length === 0) {
return true;
if (input.forcePrompt === true) {
input.io.stderr.write(`No tables discovered for ${input.connectionId}; edit was not saved.\n`);
}
return input.forcePrompt === true ? 'failed' : 'ready';
}
const allQualified = discovered.map((t) => `${t.schema}.${t.name}`);
@ -1328,7 +1469,7 @@ async function maybeConfigureTableScope(input: {
writeSetupSection(input.io, `Tables enabled for ${input.connectionId}`, [
`✓ ${allQualified[0]}`,
]);
return true;
return 'ready';
}
const bySchema = new Map<string, KtxTableListEntry[]>();
@ -1354,7 +1495,7 @@ async function maybeConfigureTableScope(input: {
});
if (action === 'back') {
return false;
return 'back';
}
if (action === 'all') {
@ -1370,7 +1511,10 @@ async function maybeConfigureTableScope(input: {
const suffix = t.kind === 'view' ? ' (view)' : '';
return { value: qualified, label: `${qualified}${suffix}` };
}),
initialValues: allQualified,
initialValues:
Array.isArray(existingTables) && input.forcePrompt === true
? existingTables.filter((table): table is string => typeof table === 'string' && allQualified.includes(table))
: allQualified,
required: true,
});
@ -1394,7 +1538,7 @@ async function maybeConfigureTableScope(input: {
writeSetupSection(input.io, `Tables enabled for ${input.connectionId}`, [
`✓ ${selected.length}/${discovered.length} tables enabled`,
]);
return true;
return 'ready';
}
async function ensureHistoricSqlIngestDefaults(projectDir: string): Promise<void> {
@ -1546,7 +1690,8 @@ async function validateAndScanConnection(input: {
deps: KtxSetupDatabasesDeps;
args: KtxSetupDatabasesArgs;
prompts: KtxSetupDatabasesPromptAdapter;
}): Promise<boolean> {
forceScopeAndTables?: boolean;
}): Promise<ConnectionSetupStatus> {
const testConnection = input.deps.testConnection ?? defaultTestConnection;
const scanConnection = input.deps.scanConnection ?? defaultScanConnection;
const project = await loadKtxProject({ projectDir: input.projectDir });
@ -1557,7 +1702,7 @@ async function validateAndScanConnection(input: {
if (testCode !== 0) {
flushBufferedCommandOutput(input.io, testIo);
input.io.stderr.write(`Connection test failed for ${input.connectionId}.\n`);
return false;
return 'failed';
}
const testOutput = testIo.stdoutText();
const outputDriver = normalizeDriver(readOutputValue(testOutput, 'Driver'));
@ -1566,14 +1711,24 @@ async function validateAndScanConnection(input: {
writeSetupSection(input.io, `Testing ${input.connectionId}`, testLines);
while (true) {
if (!(await maybeConfigureSchemaScope(input))) {
return false;
const schemaStatus = await maybeConfigureSchemaScope({ ...input, forcePrompt: input.forceScopeAndTables });
if (schemaStatus !== 'ready') {
return schemaStatus;
}
if (await maybeConfigureTableScope(input)) {
const tableStatus = await maybeConfigureTableScope({ ...input, forcePrompt: input.forceScopeAndTables });
if (tableStatus === 'ready') {
break;
}
if (input.forceScopeAndTables) {
return tableStatus;
}
if (tableStatus === 'failed') {
return 'failed';
}
await clearScopeConfig(input.projectDir, input.connectionId);
}
@ -1634,7 +1789,7 @@ async function validateAndScanConnection(input: {
);
}
if (scanCode !== 0) {
return false;
return 'failed';
}
}
const scanOutput = scanIo.stdoutText();
@ -1646,14 +1801,14 @@ async function validateAndScanConnection(input: {
writeSetupSection(input.io, 'Database ready', [
`${input.connectionId} · ${driverDisplay} · schema context complete`,
]);
return true;
return 'ready';
}
async function chooseDrivers(
args: KtxSetupDatabasesArgs,
io: KtxCliIo,
prompts: KtxSetupDatabasesPromptAdapter,
options?: { hasPrimarySources?: boolean },
options?: { hasPrimarySources?: boolean; initialDrivers?: KtxSetupDatabaseDriver[] },
): Promise<KtxSetupDatabaseDriver[] | 'back' | 'missing-input'> {
if (args.databaseDrivers && args.databaseDrivers.length > 0) {
return [...new Set(args.databaseDrivers)];
@ -1668,10 +1823,12 @@ async function chooseDrivers(
return 'missing-input';
}
while (true) {
const initialValues = unique(options?.initialDrivers ?? []);
const choices = await prompts.multiselect({
message: withMultiselectNavigation('Which databases should KTX connect to?'),
options: [...DRIVER_OPTIONS],
required: false,
...(initialValues.length > 0 ? { initialValues } : {}),
required: options?.hasPrimarySources === true,
});
if (choices.includes('back')) {
return 'back';
@ -1693,7 +1850,7 @@ async function chooseConnectionIdForDriver(input: {
connections: Record<string, KtxProjectConnectionConfig>;
args: KtxSetupDatabasesArgs;
prompts: KtxSetupDatabasesPromptAdapter;
}): Promise<{ kind: 'existing' | 'new'; connectionId: string } | 'back' | 'missing-input'> {
}): Promise<{ kind: 'existing' | 'new' | 'edit'; connectionId: string } | 'back' | 'missing-input'> {
if (input.args.databaseConnectionId) {
assertSafeDatabaseConnectionId(input.args.databaseConnectionId);
return { kind: 'new', connectionId: input.args.databaseConnectionId };
@ -1726,14 +1883,19 @@ async function chooseConnectionIdForDriver(input: {
options: [
...existingIds.map((connectionId) => ({
value: `existing:${connectionId}`,
label: `Use existing ${label} connection: ${connectionId}`,
label: `Keep existing ${label} connection: ${connectionId}`,
})),
{ value: 'new', label: `Add new ${label} connection` },
...existingIds.map((connectionId) => ({
value: `edit:${connectionId}`,
label: `Edit ${label} connection: ${connectionId}`,
})),
{ value: 'new', label: `Add another ${label} connection` },
{ value: 'back', label: 'Back' },
],
});
if (choice === 'back') return 'back';
if (choice.startsWith('existing:')) return { kind: 'existing', connectionId: choice.slice('existing:'.length) };
if (choice.startsWith('edit:')) return { kind: 'edit', connectionId: choice.slice('edit:'.length) };
const entered = await input.prompts.text({
message: withTextInputNavigation(connectionNamePrompt(label)),
placeholder: defaultId,
@ -1746,6 +1908,102 @@ async function chooseConnectionIdForDriver(input: {
}
}
async function choosePrimarySourceToEdit(input: {
projectDir: string;
connectionIds: string[];
prompts: KtxSetupDatabasesPromptAdapter;
}): Promise<string | 'back'> {
const project = await loadKtxProject({ projectDir: input.projectDir });
const options = input.connectionIds
.map((connectionId) => {
const driver = normalizeDriver(project.config.connections[connectionId]?.driver);
if (!driver) return null;
return { value: connectionId, label: `${connectionId} (${driverLabel(driver)})` };
})
.filter((option): option is { value: string; label: string } => option !== null);
if (options.length === 0) return 'back';
const choice = await input.prompts.select({
message: 'Database to edit',
options: [...options, { value: 'back', label: 'Back' }],
});
return choice === 'back' ? 'back' : choice;
}
async function runPrimarySourceFullEdit(input: {
projectDir: string;
connectionId: string;
args: KtxSetupDatabasesArgs;
prompts: KtxSetupDatabasesPromptAdapter;
io: KtxCliIo;
deps: KtxSetupDatabasesDeps;
}): Promise<'ready' | 'back' | 'failed'> {
const project = await loadKtxProject({ projectDir: input.projectDir });
const existing = project.config.connections[input.connectionId];
const driver = normalizeDriver(existing?.driver);
if (!existing || !driver) {
input.io.stderr.write(`Connection "${input.connectionId}" is not a configured database.\n`);
return 'failed';
}
const rollback = await createConnectionConfigRollback(input.projectDir, input.connectionId);
const replacement = await buildConnectionConfig({
driver,
connectionId: input.connectionId,
args: input.args,
prompts: input.prompts,
existingConnection: existing,
});
if (replacement === 'back') {
await rollback();
return 'back';
}
if (!replacement) {
await rollback();
return 'failed';
}
const withHistoricSql = await maybeApplyHistoricSqlConfig({
connection: replacement,
driver,
args: input.args,
prompts: input.prompts,
});
if (withHistoricSql === 'back') {
await rollback();
return 'back';
}
await writeConnectionConfig({
projectDir: input.projectDir,
connectionId: input.connectionId,
connection: withExistingPrimaryEditPromptDefaults({
previous: existing,
next: {
...withHistoricSql,
...(!Object.hasOwn(withHistoricSql, 'historicSql') && existing.historicSql !== undefined
? { historicSql: existing.historicSql }
: {}),
},
driver,
}),
});
const validated = await validateAndScanConnection({
projectDir: input.projectDir,
connectionId: input.connectionId,
io: input.io,
deps: input.deps,
args: input.args,
prompts: input.prompts,
forceScopeAndTables: true,
});
if (validated !== 'ready') {
await rollback();
return validated;
}
return 'ready';
}
export async function runKtxSetupDatabasesStep(
args: KtxSetupDatabasesArgs,
io: KtxCliIo,
@ -1768,7 +2026,18 @@ export async function runKtxSetupDatabasesStep(
prompts,
});
if (historicSqlResult === 'back') return { status: 'back', projectDir: args.projectDir };
if (!(await validateAndScanConnection({ projectDir: args.projectDir, connectionId, io, deps, args, prompts }))) {
const setupStatus = await validateAndScanConnection({
projectDir: args.projectDir,
connectionId,
io,
deps,
args,
prompts,
});
if (setupStatus === 'back') {
return { status: 'back', projectDir: args.projectDir };
}
if (setupStatus === 'failed') {
return { status: 'failed', projectDir: args.projectDir };
}
selectedConnectionIds.push(connectionId);
@ -1792,10 +2061,43 @@ export async function runKtxSetupDatabasesStep(
await markDatabasesComplete(args.projectDir, selectedConnectionIds);
return { status: 'ready', projectDir: args.projectDir, connectionIds: selectedConnectionIds };
}
if (action === 'edit') {
const connectionId = await choosePrimarySourceToEdit({
projectDir: args.projectDir,
connectionIds: selectedConnectionIds,
prompts,
});
if (connectionId === 'back') {
showConfiguredPrimaryMenu = true;
continue;
}
const editResult = await runPrimarySourceFullEdit({
projectDir: args.projectDir,
connectionId,
args,
prompts,
io,
deps,
});
if (editResult === 'back') {
showConfiguredPrimaryMenu = true;
continue;
}
if (editResult === 'failed') {
return { status: 'failed', projectDir: args.projectDir };
}
pushUniqueConnectionId(selectedConnectionIds, connectionId);
showConfiguredPrimaryMenu = true;
continue;
}
}
showConfiguredPrimaryMenu = false;
const drivers = await chooseDrivers(args, io, prompts, { hasPrimarySources: selectedConnectionIds.length > 0 });
const driverProject = await loadKtxProject({ projectDir: args.projectDir });
const drivers = await chooseDrivers(args, io, prompts, {
hasPrimarySources: selectedConnectionIds.length > 0,
initialDrivers: configuredPrimaryDrivers(driverProject.config.connections, selectedConnectionIds),
});
if (drivers === 'back') {
if (selectedConnectionIds.length > 0 && canReturnToDriverSelection && args.inputMode !== 'disabled') {
showConfiguredPrimaryMenu = true;
@ -1836,7 +2138,26 @@ export async function runKtxSetupDatabasesStep(
return { status: 'missing-input', projectDir: args.projectDir };
}
if (connectionChoice.kind === 'new') {
let connectionAlreadyValidated = false;
if (connectionChoice.kind === 'edit') {
const editResult = await runPrimarySourceFullEdit({
projectDir: args.projectDir,
connectionId: connectionChoice.connectionId,
args,
prompts,
io,
deps,
});
if (editResult === 'back') {
if (!canReturnToDriverSelection) return { status: 'back', projectDir: args.projectDir };
returnToDriverSelection = true;
break;
}
if (editResult === 'failed') {
return { status: 'failed', projectDir: args.projectDir };
}
connectionAlreadyValidated = true;
} else if (connectionChoice.kind === 'new') {
let connection = await buildConnectionConfig({
driver,
connectionId: connectionChoice.connectionId,
@ -1929,16 +2250,22 @@ export async function runKtxSetupDatabasesStep(
}
let connectionSkipped = false;
while (
!(await validateAndScanConnection({
projectDir: args.projectDir,
connectionId: connectionChoice.connectionId,
io,
deps,
args,
prompts,
}))
) {
let setupStatus: ConnectionSetupStatus = connectionAlreadyValidated
? 'ready'
: await validateAndScanConnection({
projectDir: args.projectDir,
connectionId: connectionChoice.connectionId,
io,
deps,
args,
prompts,
});
while (!connectionAlreadyValidated && setupStatus !== 'ready') {
if (setupStatus === 'back') {
if (!canReturnToDriverSelection) return { status: 'back', projectDir: args.projectDir };
returnToDriverSelection = true;
break;
}
if (args.inputMode === 'disabled') return { status: 'failed', projectDir: args.projectDir };
const action = await prompts.select({
message: `Database setup failed for ${connectionChoice.connectionId}`,
@ -1958,7 +2285,16 @@ export async function runKtxSetupDatabasesStep(
connectionSkipped = true;
break;
}
if (action === 're-enter') {
if (action === 'retry') {
setupStatus = await validateAndScanConnection({
projectDir: args.projectDir,
connectionId: connectionChoice.connectionId,
io,
deps,
args,
prompts,
});
} else if (action === 're-enter') {
const connection = await buildConnectionConfig({
driver,
connectionId: connectionChoice.connectionId,
@ -1994,6 +2330,14 @@ export async function runKtxSetupDatabasesStep(
connectionId: connectionChoice.connectionId,
connection: withContextDepth,
});
setupStatus = await validateAndScanConnection({
projectDir: args.projectDir,
connectionId: connectionChoice.connectionId,
io,
deps,
args,
prompts,
});
}
}
if (returnToDriverSelection) break;

14
packages/cli/src/setup-embeddings.test.ts
View file

@ -319,14 +319,14 @@ describe('setup embeddings step', () => {
projectDir: tempDir,
inputMode: 'disabled',
embeddingBackend: 'openai',
embeddingApiKeyEnv: 'OPENAI_API_KEY',
embeddingApiKeyEnv: 'OPENAI_API_KEY', // pragma: allowlist secret
cliVersion: '0.2.0',
runtimeInstallPolicy: 'auto',
skipEmbeddings: false,
},
io.io,
{
env: { OPENAI_API_KEY: 'sk-openai-test' },
env: { OPENAI_API_KEY: 'sk-openai-test' }, // pragma: allowlist secret
healthCheck,
},
);
@ -336,14 +336,14 @@ describe('setup embeddings step', () => {
backend: 'openai',
model: 'text-embedding-3-small',
dimensions: 1536,
openai: { apiKey: 'sk-openai-test' },
openai: { apiKey: 'sk-openai-test' }, // pragma: allowlist secret
});
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
expect(config.ingest.embeddings).toMatchObject({
backend: 'openai',
model: 'text-embedding-3-small',
dimensions: 1536,
openai: { api_key: 'env:OPENAI_API_KEY' },
openai: { api_key: 'env:OPENAI_API_KEY' }, // pragma: allowlist secret
});
expect(io.stdout()).not.toContain('sk-openai-test');
});
@ -367,7 +367,7 @@ describe('setup embeddings step', () => {
io.io,
{
prompts,
env: { OPENAI_API_KEY: 'sk-openai-test' },
env: { OPENAI_API_KEY: 'sk-openai-test' }, // pragma: allowlist secret
healthCheck,
ensureLocalEmbeddings: vi.fn(async () => managedDaemon()),
},
@ -384,7 +384,7 @@ describe('setup embeddings step', () => {
backend: 'openai',
model: 'text-embedding-3-small',
dimensions: 1536,
openai: { apiKey: 'sk-openai-test' },
openai: { apiKey: 'sk-openai-test' }, // pragma: allowlist secret
});
expect(prompts.select).toHaveBeenCalledWith(
expect.objectContaining({
@ -478,7 +478,7 @@ describe('setup embeddings step', () => {
},
makeIo().io,
{
env: { OPENAI_API_KEY: 'sk-openai-test' },
env: { OPENAI_API_KEY: 'sk-openai-test' }, // pragma: allowlist secret
healthCheck,
},
),

87
packages/cli/src/setup-models.test.ts
View file

@ -7,10 +7,8 @@ import {
BUNDLED_ANTHROPIC_MODELS,
fetchAnthropicModels,
type KtxSetupModelPromptAdapter,
runKtxSetupGcloudApplicationDefaultAuth,
runKtxSetupAnthropicModelStep,
} from './setup-models.js';
import type { KtxCliIo } from './cli-runtime.js';
function makeIo() {
let stdout = '';
@ -34,6 +32,17 @@ function makeIo() {
};
}
function makeSpinnerEvents() {
const events: string[] = [];
const spinner = vi.fn(() => ({
start: (msg: string) => events.push(`start:${msg}`),
message: (msg: string) => events.push(`message:${msg}`),
stop: (msg: string) => events.push(`stop:${msg}`),
error: (msg: string) => events.push(`error:${msg}`),
}));
return { events, spinner };
}
function makePromptAdapter(options: {
providerChoice?: string;
selectValues?: string[];
@ -191,6 +200,7 @@ describe('setup Anthropic model step', () => {
it('configures env credentials, selected model, prompt caching, and llm completion state', async () => {
const io = makeIo();
const { events: spinnerEvents, spinner } = makeSpinnerEvents();
const result = await runKtxSetupAnthropicModelStep(
{
projectDir: tempDir,
@ -203,6 +213,7 @@ describe('setup Anthropic model step', () => {
{
env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, // pragma: allowlist secret
healthCheck: vi.fn(async () => ({ ok: true as const })),
spinner,
},
);
@ -219,6 +230,10 @@ describe('setup Anthropic model step', () => {
expect(config.scan.enrichment.mode).toBe('llm');
expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:');
expect((await readKtxSetupState(tempDir)).completed_steps).toContain('llm');
expect(spinnerEvents).toEqual([
'start:Checking Anthropic API LLM (claude-sonnet-4-6).',
'stop:LLM test passed (Anthropic API, claude-sonnet-4-6)',
]);
expect(io.stdout()).toContain('LLM ready: yes');
expect(io.stdout()).not.toContain('sk-ant-test');
});
@ -226,6 +241,7 @@ describe('setup Anthropic model step', () => {
it('configures Vertex AI provider, selected model, prompt caching, and llm completion state', async () => {
const io = makeIo();
const healthCheck = vi.fn(async () => ({ ok: true as const }));
const { events: spinnerEvents, spinner } = makeSpinnerEvents();
const result = await runKtxSetupAnthropicModelStep(
{
@ -238,7 +254,7 @@ describe('setup Anthropic model step', () => {
skipLlm: false,
},
io.io,
{ env: {}, healthCheck },
{ env: {}, healthCheck, spinner },
);
expect(result.status).toBe('ready');
@ -260,13 +276,16 @@ describe('setup Anthropic model step', () => {
expect(config.scan.enrichment.mode).toBe('llm');
expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:');
expect((await readKtxSetupState(tempDir)).completed_steps).toContain('llm');
expect(spinnerEvents).toEqual([
'start:Checking Vertex AI LLM (claude-sonnet-4-6).',
'stop:LLM test passed (Vertex AI, claude-sonnet-4-6)',
]);
expect(io.stdout()).toContain('LLM ready: yes (claude-sonnet-4-6)');
});
it('can run gcloud auth for Vertex AI and infer project and default location', async () => {
it('uses existing Vertex AI credentials without offering to run gcloud auth', async () => {
const io = makeIo();
const prompts = makePromptAdapter({ selectValues: ['vertex', 'gcloud', 'local-gcp-project', 'claude-sonnet-4-6'] });
const runGcloudAuth = vi.fn(async () => ({ ok: true as const }));
const prompts = makePromptAdapter({ selectValues: ['vertex', 'existing', 'local-gcp-project', 'claude-sonnet-4-6'] });
const readGcloudProject = vi.fn(async () => 'local-gcp-project');
const listGcloudProjects = vi.fn(async () => [
{ projectId: 'local-gcp-project', name: 'Local project' },
@ -280,7 +299,6 @@ describe('setup Anthropic model step', () => {
{
prompts,
env: {},
runGcloudAuth,
readGcloudProject,
listGcloudProjects,
healthCheck,
@ -288,7 +306,15 @@ describe('setup Anthropic model step', () => {
);
expect(result.status).toBe('ready');
expect(runGcloudAuth).toHaveBeenCalledWith(io.io);
expect(prompts.select).toHaveBeenCalledWith(
expect.objectContaining({
message: expect.stringContaining('How should KTX authenticate with Google Vertex AI?'),
options: [
{ value: 'existing', label: 'Use existing gcloud/Application Default Credentials' },
{ value: 'back', label: 'Back' },
],
}),
);
expect(readGcloudProject).toHaveBeenCalled();
expect(listGcloudProjects).toHaveBeenCalled();
expect(prompts.text).not.toHaveBeenCalled();
@ -303,6 +329,22 @@ describe('setup Anthropic model step', () => {
],
}),
);
expect(prompts.select).toHaveBeenCalledWith(
expect.objectContaining({
message: expect.stringContaining('Which Anthropic model should KTX use?'),
options: [
{ value: 'claude-opus-4-7', label: 'Claude Opus 4.7' },
{ value: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6' },
{ value: 'claude-opus-4-6', label: 'Claude Opus 4.6' },
{ value: 'claude-opus-4-5', label: 'Claude Opus 4.5' },
{ value: 'claude-haiku-4-5', label: 'Claude Haiku 4.5' },
{ value: 'claude-sonnet-4-5', label: 'Claude Sonnet 4.5' },
{ value: 'claude-opus-4-1', label: 'Claude Opus 4.1' },
{ value: 'manual', label: 'Enter a model ID manually' },
{ value: 'back', label: 'Back' },
],
}),
);
expect(healthCheck).toHaveBeenCalledWith({
backend: 'vertex',
vertex: { project: 'local-gcp-project', location: 'us-east5' },
@ -415,35 +457,6 @@ describe('setup Anthropic model step', () => {
);
});
it('runs only gcloud application-default login for Vertex AI auth', async () => {
const io = makeIo();
const runGcloud = vi.fn(async () => ({ ok: true as const }));
await expect(runKtxSetupGcloudApplicationDefaultAuth(io.io, runGcloud)).resolves.toEqual({ ok: true });
expect(runGcloud).toHaveBeenCalledTimes(1);
expect(runGcloud).toHaveBeenCalledWith(['auth', 'application-default', 'login'], expect.anything());
expect(runGcloud).not.toHaveBeenCalledWith(['auth', 'login'], expect.anything());
expect(io.stdout()).toContain('gcloud auth application-default login');
expect(io.stdout()).not.toContain('gcloud auth login');
});
it('indents gcloud auth output inside the setup gutter', async () => {
const io = makeIo();
const runGcloud = vi.fn(async (_args: string[], commandIo: KtxCliIo) => {
commandIo.stdout.write('Your browser has been opened to visit:\n\n https://accounts.example/auth\n');
commandIo.stderr.write('Credentials saved to file: [/tmp/application_default_credentials.json]\n');
return { ok: true as const };
});
await expect(runKtxSetupGcloudApplicationDefaultAuth(io.io, runGcloud)).resolves.toEqual({ ok: true });
expect(io.stdout()).toContain('│ Your browser has been opened to visit:');
expect(io.stdout()).toContain('│ https://accounts.example/auth');
expect(io.stderr()).toContain('│ Credentials saved to file: [/tmp/application_default_credentials.json]');
expect(io.stdout()).not.toContain('\nYour browser has been opened');
});
it('explains common Vertex AI Forbidden health-check causes', async () => {
const io = makeIo();

176
packages/cli/src/setup-models.ts
View file

@ -1,4 +1,4 @@
import { execFile, spawn } from 'node:child_process';
import { execFile } from 'node:child_process';
import { writeFile } from 'node:fs/promises';
import { promisify } from 'node:util';
import { resolveLocalKtxLlmConfig } from '@ktx/context';
@ -11,6 +11,7 @@ import {
serializeKtxProjectConfig,
} from '@ktx/context/project';
import { type KtxLlmConfig, type KtxLlmHealthCheckResult, runKtxLlmHealthCheck } from '@ktx/llm';
import { createClackSpinner, type KtxCliSpinner } from './clack.js';
import type { KtxCliIo } from './cli-runtime.js';
import { withTextInputNavigation } from './prompt-navigation.js';
import { envCredentialReference, writeProjectLocalSecretReference } from './setup-secrets.js';
@ -61,9 +62,9 @@ export interface KtxSetupModelDeps {
prompts?: KtxSetupModelPromptAdapter;
listModels?: (apiKey: string) => Promise<AnthropicModelChoice[]>;
healthCheck?: (config: KtxLlmConfig) => Promise<KtxLlmHealthCheckResult>;
runGcloudAuth?: (io: KtxCliIo) => Promise<GcloudAuthResult>;
readGcloudProject?: () => Promise<string | undefined>;
listGcloudProjects?: () => Promise<GcloudProjectChoice[]>;
spinner?: () => KtxCliSpinner;
}
export const BUNDLED_ANTHROPIC_MODEL_REGISTRY_VERSION = '2026-05-07';
@ -74,6 +75,16 @@ export const BUNDLED_ANTHROPIC_MODELS: AnthropicModelChoice[] = [
{ id: 'claude-haiku-4-5', label: 'Claude Haiku 4.5', recommended: false },
];
const VERTEX_ANTHROPIC_MODELS: AnthropicModelChoice[] = [
{ id: 'claude-opus-4-7', label: 'Claude Opus 4.7', recommended: false },
{ id: 'claude-sonnet-4-6', label: 'Claude Sonnet 4.6', recommended: false },
{ id: 'claude-opus-4-6', label: 'Claude Opus 4.6', recommended: false },
{ id: 'claude-opus-4-5', label: 'Claude Opus 4.5', recommended: false },
{ id: 'claude-haiku-4-5', label: 'Claude Haiku 4.5', recommended: false },
{ id: 'claude-sonnet-4-5', label: 'Claude Sonnet 4.5', recommended: false },
{ id: 'claude-opus-4-1', label: 'Claude Opus 4.1', recommended: false },
];
const HIDDEN_ANTHROPIC_MODEL_PATTERNS = [
/^claude-sonnet-4$/i,
/^claude-opus-4$/i,
@ -91,8 +102,8 @@ const ANTHROPIC_MODEL_PROMPT_CONTEXT =
'into semantic-layer sources and wiki context.';
const VERTEX_AUTH_PROMPT_CONTEXT =
'KTX can use Google Cloud Application Default Credentials for local Vertex AI access. This opens the normal ' +
'gcloud browser login flow and does not store Google credentials in ktx.yaml.';
'KTX uses Google Cloud Application Default Credentials for local Vertex AI access and does not store Google ' +
'credentials in ktx.yaml. If needed, run gcloud auth application-default login before continuing.';
const VERTEX_PROJECT_PROMPT_CONTEXT =
'KTX stores the selected Google Cloud project ID in ktx.yaml and uses Application Default Credentials for ' +
'access. Project visibility depends on the signed-in Google account and organization permissions.';
@ -137,94 +148,17 @@ type VertexConfigChoice =
}
| { status: 'back' | 'missing-input' };
type VertexAuthChoice = { status: 'ready' } | { status: 'back' | 'missing-input' };
type VertexAuthChoice = { status: 'ready' } | { status: 'back' };
export type GcloudAuthResult = { ok: true } | { ok: false; message: string };
interface GcloudProjectChoice {
projectId: string;
name?: string;
}
type GcloudCommandRunner = (args: string[], io: KtxCliIo) => Promise<GcloudAuthResult>;
function createPromptAdapter(): KtxSetupModelPromptAdapter {
return createKtxSetupPromptAdapter({ selectCancelValue: 'back' });
}
function createIndentedCommandIo(io: KtxCliIo): KtxCliIo {
const indentedWriter = (write: (chunk: string) => void) => {
let atLineStart = true;
return (chunk: string) => {
for (const char of chunk) {
if (atLineStart) {
write('│ ');
atLineStart = false;
}
write(char);
if (char === '\n') {
atLineStart = true;
}
}
};
};
return {
stdout: {
isTTY: io.stdout.isTTY,
columns: io.stdout.columns,
write: indentedWriter((chunk) => io.stdout.write(chunk)),
},
stderr: {
write: indentedWriter((chunk) => io.stderr.write(chunk)),
},
};
}
function runInteractiveGcloud(args: string[], io: KtxCliIo): Promise<GcloudAuthResult> {
return new Promise((resolve) => {
let settled = false;
const child = spawn('gcloud', args, { stdio: ['inherit', 'pipe', 'pipe'] });
child.stdout?.on('data', (chunk: Buffer) => {
io.stdout.write(chunk.toString('utf8'));
});
child.stderr?.on('data', (chunk: Buffer) => {
io.stderr.write(chunk.toString('utf8'));
});
child.on('error', (error: NodeJS.ErrnoException) => {
if (settled) {
return;
}
settled = true;
if (error.code === 'ENOENT') {
resolve({ ok: false, message: 'gcloud CLI was not found on PATH.' });
return;
}
resolve({ ok: false, message: error.message });
});
child.on('close', (code, signal) => {
if (settled) {
return;
}
settled = true;
if (code === 0) {
resolve({ ok: true });
return;
}
resolve({
ok: false,
message: signal ? `gcloud exited after signal ${signal}.` : `gcloud exited with code ${code ?? 'unknown'}.`,
});
});
});
}
export async function runKtxSetupGcloudApplicationDefaultAuth(
io: KtxCliIo,
runGcloud: GcloudCommandRunner = runInteractiveGcloud,
): Promise<GcloudAuthResult> {
io.stdout.write('│ Running gcloud auth application-default login...\n');
return await runGcloud(['auth', 'application-default', 'login'], createIndentedCommandIo(io));
}
async function defaultReadGcloudProject(): Promise<string | undefined> {
try {
const { stdout } = await execFileAsync('gcloud', ['config', 'get-value', 'project'], { encoding: 'utf8' });
@ -374,6 +308,53 @@ function buildVertexHealthConfig(vertex: { project?: string; location: string },
};
}
type LlmHealthProvider = 'Anthropic API' | 'Vertex AI';
function llmHealthCheckStartText(provider: LlmHealthProvider, model: string): string {
return `Checking ${provider} LLM (${model}).`;
}
function startLlmHealthCheckProgress(
spinner: KtxCliSpinner,
message: string,
): { succeed(msg: string): void; fail(msg: string): void } {
spinner.start(message);
return {
succeed(msg: string) {
spinner.stop(msg);
},
fail(msg: string) {
spinner.error(msg);
},
};
}
async function runLlmHealthCheckWithProgress(
config: KtxLlmConfig,
provider: LlmHealthProvider,
model: string,
healthCheck: (config: KtxLlmConfig) => Promise<KtxLlmHealthCheckResult>,
deps: KtxSetupModelDeps,
): Promise<KtxLlmHealthCheckResult> {
const progress = startLlmHealthCheckProgress(
(deps.spinner ?? createClackSpinner)(),
llmHealthCheckStartText(provider, model),
);
let health: KtxLlmHealthCheckResult;
try {
health = await healthCheck(config);
} catch (error) {
progress.fail('LLM test failed');
throw error;
}
if (health.ok) {
progress.succeed(`LLM test passed (${provider}, ${model})`);
} else {
progress.fail('LLM test failed');
}
return health;
}
function formatVertexHealthFailure(message: string, vertex: { project?: string; location: string }): string {
const trimmed = message.trim() || 'unknown error';
if (!/(forbidden|permission|permission_denied|403)/i.test(trimmed)) {
@ -516,7 +497,6 @@ async function chooseBackend(
async function chooseVertexAuth(
args: KtxSetupModelArgs,
io: KtxCliIo,
deps: KtxSetupModelDeps,
): Promise<VertexAuthChoice> {
if (args.inputMode === 'disabled' || args.vertexProject || args.vertexLocation) {
@ -527,7 +507,6 @@ async function chooseVertexAuth(
const choice = await prompts.select({
message: `How should KTX authenticate with Google Vertex AI?\n\n${VERTEX_AUTH_PROMPT_CONTEXT}`,
options: [
{ value: 'gcloud', label: 'Run gcloud Application Default Credentials login' },
{ value: 'existing', label: 'Use existing gcloud/Application Default Credentials' },
{ value: 'back', label: 'Back' },
],
@ -535,15 +514,6 @@ async function chooseVertexAuth(
if (choice === 'back') {
return { status: 'back' };
}
if (choice !== 'gcloud') {
return { status: 'ready' };
}
const result = await (deps.runGcloudAuth ?? runKtxSetupGcloudApplicationDefaultAuth)(io);
if (!result.ok) {
io.stderr.write(`gcloud authentication failed: ${result.message}\n`);
return { status: 'missing-input' };
}
return { status: 'ready' };
}
@ -799,7 +769,7 @@ async function chooseVertexModel(args: KtxSetupModelArgs, io: KtxCliIo, deps: Kt
return { status: 'missing-input' };
}
const selectableModels = BUNDLED_ANTHROPIC_MODELS.filter(isSelectableAnthropicModel);
const selectableModels = VERTEX_ANTHROPIC_MODELS.filter(isSelectableAnthropicModel);
const prompts = deps.prompts ?? createPromptAdapter();
const choice = await prompts.select({
message: `Which Anthropic model should KTX use?\n\n${ANTHROPIC_MODEL_PROMPT_CONTEXT}`,
@ -901,7 +871,7 @@ export async function runKtxSetupAnthropicModelStep(
: attemptArgs;
if (backendChoice.backend === 'vertex') {
const auth = await chooseVertexAuth(backendArgs, io, deps);
const auth = await chooseVertexAuth(backendArgs, deps);
if (auth.status === 'back' && backendChoice.prompted) {
attemptArgs = buildInteractiveRetryArgs(args);
continue;
@ -931,7 +901,13 @@ export async function runKtxSetupAnthropicModelStep(
return { status: model.status, projectDir: args.projectDir };
}
const health = await healthCheck(buildVertexHealthConfig(vertex.values, model.model));
const health = await runLlmHealthCheckWithProgress(
buildVertexHealthConfig(vertex.values, model.model),
'Vertex AI',
model.model,
healthCheck,
deps,
);
if (health.ok) {
await persistLlmConfig(args.projectDir, { backend: 'vertex', vertex: vertex.refs }, model.model);
io.stdout.write(`│ LLM ready: yes (${model.model})\n`);
@ -973,7 +949,13 @@ export async function runKtxSetupAnthropicModelStep(
return { status: model.status, projectDir: args.projectDir };
}
const health = await healthCheck(buildAnthropicHealthConfig(credential.value, model.model));
const health = await runLlmHealthCheckWithProgress(
buildAnthropicHealthConfig(credential.value, model.model),
'Anthropic API',
model.model,
healthCheck,
deps,
);
if (health.ok) {
await persistLlmConfig(args.projectDir, { backend: 'anthropic', credentialRef: credential.ref }, model.model);
io.stdout.write(`│ LLM ready: yes (${model.model})\n`);

321
packages/cli/src/setup-sources.test.ts
View file

@ -98,7 +98,7 @@ describe('setup sources step', () => {
...config,
connections: {
...config.connections,
warehouse: { driver: 'postgres', url: 'env:DATABASE_URL', readonly: true },
warehouse: { driver: 'postgres', url: 'env:DATABASE_URL' },
},
setup: {
...config.setup,
@ -486,7 +486,6 @@ describe('setup sources step', () => {
driver: 'snowflake',
account: 'acme',
database: 'analytics',
readonly: true,
});
const cases: Array<{
@ -787,7 +786,7 @@ describe('setup sources step', () => {
expect(testPrompts.text).toHaveBeenCalledTimes(4);
});
it('adds a dbt source connection without adapter allow-list entries', async () => {
it('adds a dbt source connection and enables its adapter', async () => {
await addPrimarySource();
const validateDbt = vi.fn(async () => ({ ok: true as const, detail: 'project=analytics schemas=2' }));
@ -810,8 +809,7 @@ describe('setup sources step', () => {
const configText = await readFile(join(projectDir, 'ktx.yaml'), 'utf-8');
expect(configText).not.toContain('live-database');
expect(configText).not.toContain('historic-sql');
expect(configText).not.toMatch(/^\s+adapters:/m);
expect((await readConfig()).ingest.adapters).toEqual([]);
expect((await readConfig()).ingest.adapters).toEqual(['dbt']);
});
it('lets interactive setup retry or continue after initial source ingest fails', async () => {
@ -899,6 +897,7 @@ describe('setup sources step', () => {
message: 'Configure dbt',
options: [
{ value: 'existing:dbt-main', label: 'Use existing dbt connection: dbt-main' },
{ value: 'edit:dbt-main', label: 'Edit existing dbt connection: dbt-main' },
{ value: 'new', label: 'Add new dbt connection' },
{ value: 'back', label: 'Back' },
],
@ -1026,6 +1025,10 @@ describe('setup sources step', () => {
value: `existing:${testCase.connectionId}`,
label: `Use existing ${testCase.expectedLabel} connection: ${testCase.connectionId}`,
},
{
value: `edit:${testCase.connectionId}`,
label: `Edit existing ${testCase.expectedLabel} connection: ${testCase.connectionId}`,
},
{ value: 'new', label: `Add new ${testCase.expectedLabel} connection` },
{ value: 'back', label: 'Back' },
],
@ -1034,6 +1037,314 @@ describe('setup sources step', () => {
}
});
it('edits an existing Notion source and reopens the page picker with stored pages selected', async () => {
await addPrimarySource();
await addConnection('notion-main', {
driver: 'notion',
auth_token_ref: 'env:NOTION_TOKEN',
crawl_mode: 'selected_roots',
root_page_ids: ['old-page'],
root_database_ids: [],
root_data_source_ids: [],
});
const validateNotion = vi.fn(async () => ({ ok: true as const, detail: 'roots=1' }));
const pickNotionRootPages = vi.fn(async () => ({ kind: 'selected' as const, rootPageIds: ['new-page'] }));
const testPrompts = prompts({
multiselect: [['notion']],
select: ['edit:notion-main', 'keep', 'selected_roots', 'done'],
});
await expect(
runKtxSetupSourcesStep(
{ projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false },
makeIo().io,
{
prompts: testPrompts,
validateNotion,
pickNotionRootPages,
},
),
).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['notion-main'] });
expect(testPrompts.select).toHaveBeenCalledWith({
message: 'How should KTX find your Notion integration token?',
options: [
{ value: 'keep', label: 'Keep existing credential' },
{ value: 'env', label: 'Use NOTION_TOKEN from the environment' },
{ value: 'paste', label: 'Paste a key and save it as a local secret file' },
{ value: 'back', label: 'Back' },
],
});
expect(pickNotionRootPages).toHaveBeenCalledWith(
{
connectionId: 'notion-main',
connection: expect.objectContaining({
driver: 'notion',
auth_token_ref: 'env:NOTION_TOKEN',
crawl_mode: 'selected_roots',
root_page_ids: ['old-page'],
}),
},
expect.anything(),
);
expect((await readConfig()).connections['notion-main']).toMatchObject({
driver: 'notion',
auth_token_ref: 'env:NOTION_TOKEN',
crawl_mode: 'selected_roots',
root_page_ids: ['new-page'],
});
});
it('edits an existing Metabase source with the current URL and credential as defaults', async () => {
await addPrimarySource();
await addConnection('metabase-main', {
driver: 'metabase',
api_url: 'https://metabase-old.example.com',
api_key_ref: 'env:METABASE_API_KEY', // pragma: allowlist secret
mappings: {
databaseMappings: { '1': 'warehouse' },
syncEnabled: { '1': true },
syncMode: 'ALL',
},
});
const testPrompts = prompts({
multiselect: [['metabase']],
select: ['edit:metabase-main', 'keep', 'done'],
text: ['https://metabase-new.example.com'],
});
const discoverMetabaseDatabases = vi.fn(async () => [
{ id: 2, name: 'Analytics', engine: 'postgres', host: 'db.example.com', dbName: 'analytics' },
]);
await expect(
runKtxSetupSourcesStep(
{ projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false },
makeIo().io,
{
prompts: testPrompts,
discoverMetabaseDatabases,
validateMetabase: vi.fn(async () => ({ ok: true as const, detail: 'mapping validated' })),
runMapping: vi.fn(async () => 0),
},
),
).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['metabase-main'] });
expect(testPrompts.text).toHaveBeenCalledWith({
message: textInputPrompt('Metabase URL'),
initialValue: 'https://metabase-old.example.com',
});
expect(testPrompts.select).toHaveBeenCalledWith({
message: 'How should KTX find your Metabase API key?',
options: [
{ value: 'keep', label: 'Keep existing credential' },
{ value: 'env', label: 'Use METABASE_API_KEY from the environment' },
{ value: 'paste', label: 'Paste a key and save it as a local secret file' },
{ value: 'back', label: 'Back' },
],
});
expect(discoverMetabaseDatabases).toHaveBeenCalledWith({
sourceUrl: 'https://metabase-new.example.com',
sourceApiKeyRef: 'env:METABASE_API_KEY',
sourceConnectionId: 'metabase-main',
});
expect((await readConfig()).connections['metabase-main']).toMatchObject({
driver: 'metabase',
api_url: 'https://metabase-new.example.com',
api_key_ref: 'env:METABASE_API_KEY',
mappings: {
databaseMappings: { '2': 'warehouse' },
syncEnabled: { '2': true },
syncMode: 'ALL',
},
});
});
it('rolls back an edited context source when validation fails', async () => {
await addPrimarySource();
await addConnection('dbt-main', {
driver: 'dbt',
source_dir: '/repo/existing-dbt',
project_name: 'analytics',
});
const validateDbt = vi.fn(async () => ({ ok: false as const, message: 'dbt project not found' }));
const testPrompts = prompts({
multiselect: [['dbt']],
select: ['edit:dbt-main', 'path'],
text: ['/repo/new-dbt', ''],
});
await expect(
runKtxSetupSourcesStep(
{ projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false },
makeIo().io,
{
prompts: testPrompts,
validateDbt,
},
),
).resolves.toEqual({ status: 'failed', projectDir });
expect(validateDbt).toHaveBeenCalledWith(expect.objectContaining({
driver: 'dbt',
source_dir: '/repo/new-dbt',
}));
const config = await readConfig();
expect(config.connections['dbt-main']).toMatchObject({
driver: 'dbt',
source_dir: '/repo/existing-dbt',
project_name: 'analytics',
});
expect(config.ingest.adapters).not.toContain('dbt');
});
it('lets git-backed context source edits keep the existing repo credential', async () => {
await addPrimarySource();
await addConnection('metricflow-main', {
driver: 'metricflow',
metricflow: {
repoUrl: 'https://github.com/acme/private-metricflow',
branch: 'main',
path: 'metrics',
auth_token_ref: 'env:METRICFLOW_REPO_TOKEN', // pragma: allowlist secret
},
});
const testGitRepo = vi.fn(async () => ({ ok: false as const, error: 'authentication required' }));
const testPrompts = prompts({
multiselect: [['metricflow']],
select: ['edit:metricflow-main', 'git', 'keep', 'done'],
text: ['https://github.com/acme/private-metricflow', 'main', 'metrics'],
});
await expect(
runKtxSetupSourcesStep(
{ projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false },
makeIo().io,
{
prompts: testPrompts,
testGitRepo,
validateMetricflow: vi.fn(async () => ({ ok: true as const, detail: 'metrics=1' })),
},
),
).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['metricflow-main'] });
expect(testPrompts.select).toHaveBeenCalledWith({
message: 'This MetricFlow repo requires authentication.',
options: [
{ value: 'keep', label: 'Keep existing credential' },
{ value: 'env', label: 'Use GITHUB_TOKEN from the environment' },
{ value: 'paste', label: 'Paste a token and save it as a local secret file' },
{ value: 'skip', label: 'Skip — try without authentication' },
{ value: 'back', label: 'Back' },
],
});
expect((await readConfig()).connections['metricflow-main']).toMatchObject({
driver: 'metricflow',
metricflow: {
repoUrl: 'https://github.com/acme/private-metricflow',
branch: 'main',
path: 'metrics',
auth_token_ref: 'env:METRICFLOW_REPO_TOKEN',
},
});
});
it('edits an existing context source from the configured-source follow-up menu', async () => {
await addPrimarySource();
await addConnection('dbt-main', {
driver: 'dbt',
source_dir: '/repo/existing-dbt',
project_name: 'analytics',
});
const validateDbt = vi.fn(async () => ({ ok: true as const, detail: 'project=analytics schemas=2' }));
const testPrompts = prompts({
multiselect: [['dbt']],
select: ['existing:dbt-main', 'edit', 'dbt-main', 'path', 'done'],
text: ['/repo/edited-dbt', ''],
});
await expect(
runKtxSetupSourcesStep(
{ projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false },
makeIo().io,
{
prompts: testPrompts,
validateDbt,
},
),
).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['dbt-main'] });
expect(testPrompts.select).toHaveBeenCalledWith({
message: '1 context source configured (dbt-main). Add another?',
options: [
{ value: 'done', label: 'Done — continue to context build' },
{ value: 'edit', label: 'Edit an existing context source' },
{ value: 'add', label: 'Add another context source' },
],
});
expect(testPrompts.select).toHaveBeenCalledWith({
message: 'Context source to edit',
options: [
{ value: 'dbt-main', label: 'dbt-main (dbt)' },
{ value: 'back', label: 'Back' },
],
});
expect(testPrompts.text).toHaveBeenCalledWith({
message: textInputPrompt('dbt local path'),
initialValue: '/repo/existing-dbt',
});
expect(validateDbt).toHaveBeenLastCalledWith(expect.objectContaining({
driver: 'dbt',
source_dir: '/repo/edited-dbt',
}));
expect((await readConfig()).connections['dbt-main']).toMatchObject({
driver: 'dbt',
source_dir: '/repo/edited-dbt',
project_name: 'analytics',
});
});
it('backs out of editing an existing context source to the source connection menu', async () => {
await addPrimarySource();
await addConnection('dbt-main', {
driver: 'dbt',
source_dir: '/repo/existing-dbt',
project_name: 'analytics',
});
const validateDbt = vi.fn(async () => ({ ok: true as const, detail: 'project=analytics schemas=2' }));
const testPrompts = prompts({
multiselect: [['dbt']],
select: ['edit:dbt-main', 'back', 'existing:dbt-main'],
});
await expect(
runKtxSetupSourcesStep(
{ projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false },
makeIo().io,
{
prompts: testPrompts,
validateDbt,
},
),
).resolves.toEqual({ status: 'ready', projectDir, connectionIds: ['dbt-main'] });
expect(
vi
.mocked(testPrompts.select)
.mock.calls.map(([options]) => options.message)
.filter((message) => message === 'Configure dbt'),
).toHaveLength(2);
expect(validateDbt).toHaveBeenCalledWith({
driver: 'dbt',
source_dir: '/repo/existing-dbt',
project_name: 'analytics',
});
expect((await readConfig()).connections['dbt-main']).toMatchObject({
driver: 'dbt',
source_dir: '/repo/existing-dbt',
project_name: 'analytics',
});
});
it('lets Escape from dbt git URL return to source location selection', async () => {
await addPrimarySource();
const validateDbt = vi.fn(async () => ({ ok: true as const, detail: 'project=analytics schemas=2' }));

518
packages/cli/src/setup-sources.ts
View file

@ -156,6 +156,10 @@ function sourceLabel(source: KtxSetupSourceType): string {
return SOURCE_LABELS[source];
}
function sourceAdapter(source: KtxSetupSourceType): string {
return source;
}
function connectionNamePrompt(label: string): string {
return `Name this ${label} connection\nKTX will use this short name in commands and config. You can rename it now.`;
}
@ -220,17 +224,20 @@ async function chooseSourceCredentialRef(input: {
label: string;
envName: string;
secretFileName: string;
existingRef?: string;
}): Promise<string | 'back'> {
while (true) {
const choice = await input.prompts.select({
message: `How should KTX find your ${input.label}?`,
options: [
...(input.existingRef ? [{ value: 'keep', label: 'Keep existing credential' }] : []),
{ value: 'env', label: `Use ${input.envName} from the environment` },
{ value: 'paste', label: 'Paste a key and save it as a local secret file' },
{ value: 'back', label: 'Back' },
],
});
if (choice === 'back') return 'back';
if (choice === 'keep' && input.existingRef) return input.existingRef;
if (choice === 'paste') {
const value = await input.prompts.password({ message: input.label });
if (value === undefined) continue;
@ -252,12 +259,14 @@ async function chooseGitAuthCredentialRef(input: {
projectDir: string;
source: KtxSetupSourceType;
connectionId: string;
existingRef?: string;
}): Promise<string | undefined | 'back'> {
const label = input.source === 'dbt' ? 'This' : `This ${sourceLabel(input.source)}`;
while (true) {
const choice = await input.prompts.select({
message: `${label} repo requires authentication.`,
options: [
...(input.existingRef ? [{ value: 'keep', label: 'Keep existing credential' }] : []),
{ value: 'env', label: 'Use GITHUB_TOKEN from the environment' },
{ value: 'paste', label: 'Paste a token and save it as a local secret file' },
{ value: 'skip', label: 'Skip — try without authentication' },
@ -265,6 +274,7 @@ async function chooseGitAuthCredentialRef(input: {
],
});
if (choice === 'back') return 'back';
if (choice === 'keep' && input.existingRef) return input.existingRef;
if (choice === 'skip') return undefined;
if (choice === 'paste') {
const value = await input.prompts.password({ message: 'Git access token' });
@ -309,17 +319,25 @@ async function writeSourceConnection(
projectDir: string,
connectionId: string,
connection: KtxProjectConnectionConfig,
adapter: string,
): Promise<() => Promise<void>> {
assertSafeConnectionId(connectionId);
const project = await loadKtxProject({ projectDir });
const previousConnection = project.config.connections[connectionId];
const hadPreviousConnection = previousConnection !== undefined;
const shouldRemoveAdapterOnRollback = !project.config.ingest.adapters.includes(adapter);
const config = {
...project.config,
connections: {
...project.config.connections,
[connectionId]: connection,
},
ingest: {
...project.config.ingest,
adapters: project.config.ingest.adapters.includes(adapter)
? [...project.config.ingest.adapters]
: [...project.config.ingest.adapters, adapter],
},
};
await writeFile(project.configPath, serializeKtxProjectConfig(config), 'utf-8');
return async () => {
@ -333,10 +351,31 @@ async function writeSourceConnection(
await writeProjectConfig(projectDir, {
...latest.config,
connections,
ingest: {
...latest.config.ingest,
adapters: shouldRemoveAdapterOnRollback
? latest.config.ingest.adapters.filter((candidate) => candidate !== adapter)
: latest.config.ingest.adapters,
},
});
};
}
async function ensureSourceAdapterEnabled(projectDir: string, source: KtxSetupSourceType): Promise<void> {
const adapter = sourceAdapter(source);
const project = await loadKtxProject({ projectDir });
if (project.config.ingest.adapters.includes(adapter)) {
return;
}
await writeProjectConfig(projectDir, {
...project.config,
ingest: {
...project.config.ingest,
adapters: [...project.config.ingest.adapters, adapter],
},
});
}
async function markSourcesComplete(projectDir: string): Promise<void> {
const project = await loadKtxProject({ projectDir });
await writeFile(project.configPath, serializeKtxProjectConfig(project.config), 'utf-8');
@ -760,8 +799,14 @@ interface WarehouseConnectionChoice {
type InteractiveSourceConnectionChoice =
| { kind: 'existing'; connectionId: string; connection: KtxProjectConnectionConfig }
| { kind: 'new'; args: KtxSetupSourcesArgs }
| { kind: 'edited'; connectionId: string; args: KtxSetupSourcesArgs }
| 'back';
type SourceSetupChoiceResult =
| { status: 'ready'; connectionId: string }
| { status: 'back' }
| { status: 'failed' };
async function runSourcePromptSteps(
initialState: SourcePromptState,
stepsForState: (state: SourcePromptState) => SourcePromptStep[],
@ -795,6 +840,12 @@ function resetRepoLocationFields(state: SourcePromptState): void {
delete state.sourceProjectName;
}
function sourceLocationFromArgs(args: KtxSetupSourcesArgs): SourceLocationChoice | undefined {
if (args.sourcePath) return 'path';
if (args.sourceGitUrl) return 'git';
return undefined;
}
function warehouseConnectionChoices(config: KtxProjectConfig): WarehouseConnectionChoice[] {
return Object.entries(config.connections)
.filter(([, connection]) => PRIMARY_SOURCE_DRIVERS.has(String(connection.driver ?? '').toLowerCase()))
@ -931,7 +982,7 @@ async function promptForInteractiveSource(
testGitRepo: KtxSetupSourcesDeps['testGitRepo'] = testRepoConnection,
discoverMetabaseDatabaseList?: KtxSetupSourcesDeps['discoverMetabaseDatabases'],
): Promise<KtxSetupSourcesArgs | 'back'> {
const initialState: SourcePromptState = { ...args, source };
const initialState: SourcePromptState = { ...args, source, sourceLocation: sourceLocationFromArgs(args) };
if (args.sourceConnectionId) {
initialState.sourceConnectionId = args.sourceConnectionId;
}
@ -961,7 +1012,10 @@ async function promptForInteractiveSource(
...(state.sourceLocation === 'path'
? [
async (currentState: SourcePromptState) => {
const sourcePath = await promptText(prompts, { message: `${source} local path` });
const sourcePath = await promptText(prompts, {
message: `${source} local path`,
...(currentState.sourcePath ? { initialValue: currentState.sourcePath } : {}),
});
if (sourcePath === undefined) return 'back';
currentState.sourcePath = sourcePath;
return 'next';
@ -971,13 +1025,19 @@ async function promptForInteractiveSource(
...(state.sourceLocation === 'git'
? [
async (currentState: SourcePromptState) => {
const sourceGitUrl = await promptText(prompts, { message: `${source} git URL` });
const sourceGitUrl = await promptText(prompts, {
message: `${source} git URL`,
...(currentState.sourceGitUrl ? { initialValue: currentState.sourceGitUrl } : {}),
});
if (sourceGitUrl === undefined) return 'back';
currentState.sourceGitUrl = sourceGitUrl;
return 'next';
},
async (currentState: SourcePromptState) => {
const branch = await promptText(prompts, { message: `${source} git branch`, initialValue: 'main' });
const branch = await promptText(prompts, {
message: `${source} git branch`,
initialValue: currentState.sourceBranch ?? 'main',
});
if (branch === undefined) return 'back';
currentState.sourceBranch = branch || 'main';
return 'next';
@ -998,6 +1058,7 @@ async function promptForInteractiveSource(
projectDir: args.projectDir,
source,
connectionId: currentState.sourceConnectionId ?? `${source}-main`,
existingRef: currentState.sourceAuthTokenRef,
});
if (authRef === 'back') return 'back';
if (authRef) {
@ -1071,6 +1132,7 @@ async function promptForInteractiveSource(
const subpath = await promptText(prompts, {
message: sourceSubpathPrompt(source),
placeholder: 'optional',
...(currentState.sourceSubpath ? { initialValue: currentState.sourceSubpath } : {}),
});
if (subpath === undefined) return 'back';
if (subpath) {
@ -1089,7 +1151,10 @@ async function promptForInteractiveSource(
return await runSourcePromptSteps(initialState, () => [
...connectionSteps,
async (state) => {
const sourceUrl = await promptText(prompts, { message: 'Metabase URL' });
const sourceUrl = await promptText(prompts, {
message: 'Metabase URL',
...(state.sourceUrl ? { initialValue: state.sourceUrl } : {}),
});
if (sourceUrl === undefined) return 'back';
state.sourceUrl = sourceUrl;
return 'next';
@ -1101,6 +1166,7 @@ async function promptForInteractiveSource(
label: 'Metabase API key',
envName: 'METABASE_API_KEY',
secretFileName: `${state.sourceConnectionId ?? 'metabase-main'}-api-key`,
existingRef: state.sourceApiKeyRef,
});
if (ref === 'back') return 'back';
state.sourceApiKeyRef = ref;
@ -1132,13 +1198,19 @@ async function promptForInteractiveSource(
return await runSourcePromptSteps(initialState, () => [
...connectionSteps,
async (state) => {
const sourceUrl = await promptText(prompts, { message: 'Looker base URL' });
const sourceUrl = await promptText(prompts, {
message: 'Looker base URL',
...(state.sourceUrl ? { initialValue: state.sourceUrl } : {}),
});
if (sourceUrl === undefined) return 'back';
state.sourceUrl = sourceUrl;
return 'next';
},
async (state) => {
const sourceClientId = await promptText(prompts, { message: 'Looker client id' });
const sourceClientId = await promptText(prompts, {
message: 'Looker client id',
...(state.sourceClientId ? { initialValue: state.sourceClientId } : {}),
});
if (sourceClientId === undefined) return 'back';
state.sourceClientId = sourceClientId;
return 'next';
@ -1150,6 +1222,7 @@ async function promptForInteractiveSource(
label: 'Looker client secret',
envName: 'LOOKER_CLIENT_SECRET',
secretFileName: `${state.sourceConnectionId ?? 'looker-main'}-client-secret`,
existingRef: state.sourceClientSecretRef,
});
if (ref === 'back') return 'back';
state.sourceClientSecretRef = ref;
@ -1168,6 +1241,7 @@ async function promptForInteractiveSource(
const lookerConnectionName = await promptText(prompts, {
message: 'Looker connection name',
placeholder: 'optional',
...(state.sourceTarget ? { initialValue: state.sourceTarget } : {}),
});
if (lookerConnectionName === undefined) return 'back';
if (lookerConnectionName) {
@ -1189,6 +1263,7 @@ async function promptForInteractiveSource(
label: 'Notion integration token',
envName: 'NOTION_TOKEN',
secretFileName: `${currentState.sourceConnectionId ?? 'notion-main'}-token`,
existingRef: currentState.sourceApiKeyRef,
});
if (ref === 'back') return 'back';
currentState.sourceApiKeyRef = ref;
@ -1253,6 +1328,24 @@ function existingConnectionIdsBySource(
.sort((left, right) => left.localeCompare(right));
}
function sourceTypeForConnection(connection: KtxProjectConnectionConfig): KtxSetupSourceType | null {
const driver = String(connection.driver ?? '').toLowerCase();
return SOURCE_OPTIONS.some((option) => option.value === driver) ? (driver as KtxSetupSourceType) : null;
}
function contextSourceEditTargets(connections: Record<string, KtxProjectConnectionConfig>): Array<{
connectionId: string;
source: KtxSetupSourceType;
}> {
return Object.entries(connections)
.map(([connectionId, connection]) => {
const source = sourceTypeForConnection(connection);
return source ? { connectionId, source } : null;
})
.filter((target): target is { connectionId: string; source: KtxSetupSourceType } => target !== null)
.sort((left, right) => left.connectionId.localeCompare(right.connectionId));
}
function sourceChecklistForConnections(connections: Record<string, KtxProjectConnectionConfig>): {
options: Array<{ value: KtxSetupSourceType; label: string; hint?: string }>;
initialValues: KtxSetupSourceType[];
@ -1284,6 +1377,180 @@ function defaultConnectionIdForSource(
return `${base}-${index}`;
}
function firstStringRecordEntry(value: unknown): [string, string] | undefined {
if (!isRecord(value)) return undefined;
for (const [key, raw] of Object.entries(value)) {
if (typeof raw === 'string' && raw.trim().length > 0) {
return [key, raw.trim()];
}
}
return undefined;
}
function applyRepoSourceArgs(
args: KtxSetupSourcesArgs,
input: { repoUrl?: string; sourceDir?: string; branch?: string; subpath?: string; authTokenRef?: string },
): void {
if (input.sourceDir) {
args.sourcePath = input.sourceDir;
} else if (input.repoUrl?.startsWith('file:')) {
args.sourcePath = fileURLToPath(input.repoUrl);
} else if (input.repoUrl) {
args.sourceGitUrl = input.repoUrl;
}
if (input.branch) args.sourceBranch = input.branch;
if (input.subpath) args.sourceSubpath = input.subpath;
if (input.authTokenRef) args.sourceAuthTokenRef = input.authTokenRef;
}
function sourceArgsFromExistingConnection(input: {
args: KtxSetupSourcesArgs;
source: KtxSetupSourceType;
connectionId: string;
connection: KtxProjectConnectionConfig;
}): KtxSetupSourcesArgs {
const sourceArgs: KtxSetupSourcesArgs = {
projectDir: input.args.projectDir,
inputMode: input.args.inputMode,
source: input.source,
sourceConnectionId: input.connectionId,
runInitialSourceIngest: input.args.runInitialSourceIngest,
skipSources: input.args.skipSources,
};
if (input.source === 'dbt') {
applyRepoSourceArgs(sourceArgs, {
sourceDir: stringField(input.connection.source_dir),
repoUrl: stringField(input.connection.repo_url),
branch: stringField(input.connection.branch),
subpath: stringField(input.connection.path),
authTokenRef: stringField(input.connection.auth_token_ref),
});
const profilesPath = stringField(input.connection.profiles_path);
const target = stringField(input.connection.target);
const projectName = stringField(input.connection.project_name);
if (profilesPath) sourceArgs.sourceProfilesPath = profilesPath;
if (target) sourceArgs.sourceTarget = target;
if (projectName) sourceArgs.sourceProjectName = projectName;
return sourceArgs;
}
if (input.source === 'metricflow') {
const metricflow = isRecord(input.connection.metricflow) ? input.connection.metricflow : {};
applyRepoSourceArgs(sourceArgs, {
repoUrl: stringField(metricflow.repoUrl),
branch: stringField(metricflow.branch),
subpath: stringField(metricflow.path),
authTokenRef: stringField(metricflow.auth_token_ref),
});
return sourceArgs;
}
if (input.source === 'lookml') {
applyRepoSourceArgs(sourceArgs, {
repoUrl: stringField(input.connection.repoUrl),
branch: stringField(input.connection.branch),
subpath: stringField(input.connection.path),
authTokenRef: stringField(input.connection.auth_token_ref),
});
const mappings = isRecord(input.connection.mappings) ? input.connection.mappings : {};
const expectedLookerConnectionName = stringField(mappings.expectedLookerConnectionName);
if (expectedLookerConnectionName) sourceArgs.sourceTarget = expectedLookerConnectionName;
return sourceArgs;
}
if (input.source === 'metabase') {
sourceArgs.sourceUrl = stringField(input.connection.api_url);
sourceArgs.sourceApiKeyRef = stringField(input.connection.api_key_ref);
const mappings = isRecord(input.connection.mappings) ? input.connection.mappings : {};
const databaseMapping = firstStringRecordEntry(mappings.databaseMappings);
if (databaseMapping) {
sourceArgs.metabaseDatabaseId = Number.parseInt(databaseMapping[0], 10);
sourceArgs.sourceWarehouseConnectionId = databaseMapping[1];
}
return sourceArgs;
}
if (input.source === 'looker') {
sourceArgs.sourceUrl = stringField(input.connection.base_url);
sourceArgs.sourceClientId = stringField(input.connection.client_id);
sourceArgs.sourceClientSecretRef = stringField(input.connection.client_secret_ref);
const mappings = isRecord(input.connection.mappings) ? input.connection.mappings : {};
const connectionMapping = firstStringRecordEntry(mappings.connectionMappings);
if (connectionMapping) {
sourceArgs.sourceTarget = connectionMapping[0];
sourceArgs.sourceWarehouseConnectionId = connectionMapping[1];
}
return sourceArgs;
}
sourceArgs.sourceApiKeyRef = stringField(input.connection.auth_token_ref);
sourceArgs.notionCrawlMode =
input.connection.crawl_mode === 'all_accessible' ? 'all_accessible' : 'selected_roots';
if (Array.isArray(input.connection.root_page_ids)) {
sourceArgs.notionRootPageIds = input.connection.root_page_ids.filter(
(pageId): pageId is string => typeof pageId === 'string',
);
}
return sourceArgs;
}
async function promptEditedSourceConnection(input: {
args: KtxSetupSourcesArgs;
source: KtxSetupSourceType;
connectionId: string;
connection: KtxProjectConnectionConfig;
prompts: KtxSetupSourcesPromptAdapter;
io: KtxCliIo;
testGitRepo?: KtxSetupSourcesDeps['testGitRepo'];
pickNotionRootPages?: KtxSetupSourcesDeps['pickNotionRootPages'];
discoverMetabaseDatabases?: KtxSetupSourcesDeps['discoverMetabaseDatabases'];
}): Promise<Extract<InteractiveSourceConnectionChoice, { kind: 'edited' }> | 'back'> {
const sourceArgs = await promptForInteractiveSource(
sourceArgsFromExistingConnection({
args: input.args,
source: input.source,
connectionId: input.connectionId,
connection: input.connection,
}),
input.source,
input.prompts,
input.io,
{
pickNotionRootPages: input.pickNotionRootPages,
discoverMetabaseDatabases: input.discoverMetabaseDatabases,
},
input.connectionId,
input.testGitRepo,
input.discoverMetabaseDatabases,
);
return sourceArgs === 'back'
? 'back'
: { kind: 'edited', connectionId: input.connectionId, args: sourceArgs };
}
async function chooseContextSourceToEdit(input: {
projectDir: string;
prompts: KtxSetupSourcesPromptAdapter;
}): Promise<{ connectionId: string; source: KtxSetupSourceType } | 'back'> {
const project = await loadKtxProject({ projectDir: input.projectDir });
const targets = contextSourceEditTargets(project.config.connections);
if (targets.length === 0) return 'back';
const choice = await input.prompts.select({
message: 'Context source to edit',
options: [
...targets.map((target) => ({
value: target.connectionId,
label: `${target.connectionId} (${sourceLabel(target.source)})`,
})),
{ value: 'back', label: 'Back' },
],
});
if (choice === 'back') return 'back';
const target = targets.find((candidate) => candidate.connectionId === choice);
return target ?? 'back';
}
async function chooseInteractiveSourceConnection(input: {
args: KtxSetupSourcesArgs;
source: KtxSetupSourceType;
@ -1323,6 +1590,10 @@ async function chooseInteractiveSourceConnection(input: {
value: `existing:${connectionId}`,
label: `Use existing ${label} connection: ${connectionId}`,
})),
...existingIds.map((connectionId) => ({
value: `edit:${connectionId}`,
label: `Edit existing ${label} connection: ${connectionId}`,
})),
{ value: 'new', label: `Add new ${label} connection` },
{ value: 'back', label: 'Back' },
],
@ -1336,6 +1607,28 @@ async function chooseInteractiveSourceConnection(input: {
}
continue;
}
if (choice.startsWith('edit:')) {
const connectionId = choice.slice('edit:'.length);
const connection = input.connections[connectionId];
if (!connection) {
continue;
}
const edited = await promptEditedSourceConnection({
args: input.args,
source: input.source,
connectionId,
connection,
prompts: input.prompts,
io: input.io,
testGitRepo: input.testGitRepo,
pickNotionRootPages: input.pickNotionRootPages,
discoverMetabaseDatabases: input.discoverMetabaseDatabases,
});
if (edited === 'back') {
continue;
}
return edited;
}
const sourceArgs = await promptForInteractiveSource(
input.args,
input.source,
@ -1400,6 +1693,85 @@ async function validateSource(
return await (deps.validateNotion ?? defaultValidateNotion)(args.connection);
}
async function saveValidateAndMaybeBuildSource(input: {
args: KtxSetupSourcesArgs;
source: KtxSetupSourceType;
sourceChoice: Exclude<InteractiveSourceConnectionChoice, 'back'>;
prompts: KtxSetupSourcesPromptAdapter;
io: KtxCliIo;
deps: KtxSetupSourcesDeps;
}): Promise<SourceSetupChoiceResult> {
const connectionId =
input.sourceChoice.kind === 'existing'
? input.sourceChoice.connectionId
: input.sourceChoice.kind === 'edited'
? input.sourceChoice.connectionId
: (input.sourceChoice.args.sourceConnectionId ?? `${input.source}-main`);
const connection =
input.sourceChoice.kind === 'existing'
? input.sourceChoice.connection
: buildConnection(input.source, input.sourceChoice.args);
const rollback =
input.sourceChoice.kind === 'existing'
? undefined
: await writeSourceConnection(
input.args.projectDir,
connectionId,
connection,
sourceAdapter(input.source),
);
if (input.sourceChoice.kind === 'existing') {
await ensureSourceAdapterEnabled(input.args.projectDir, input.source);
}
const validation = await validateSource(
input.source,
{ projectDir: input.args.projectDir, connectionId, connection },
input.deps,
);
if (!validation.ok) {
await rollback?.();
input.io.stderr.write(`${validation.message}\n`);
return { status: 'failed' };
}
if (input.source === 'metabase' || input.source === 'looker') {
input.prompts.log?.(`Validating ${sourceLabel(input.source)} mapping…`);
const mappingCode = await (input.deps.runMapping ?? defaultRunMapping)(
input.args.projectDir,
connectionId,
createSetupPrefixedIo(input.io),
);
if (mappingCode !== 0) {
await rollback?.();
return { status: 'failed' };
}
}
if (input.args.runInitialSourceIngest) {
const ingestResult = await runInitialSourceIngestWithRecovery({
args: input.args,
connectionId,
io: input.io,
prompts: input.prompts,
deps: input.deps,
});
if (ingestResult === 'failed') {
await rollback?.();
return { status: 'failed' };
}
if (ingestResult === 'back') {
await rollback?.();
return { status: 'back' };
}
} else {
input.io.stdout.write(`│ Context source ${connectionId} saved. It will be built during the context build step.\n`);
}
return { status: 'ready', connectionId };
}
export async function runKtxSetupSourcesStep(
args: KtxSetupSourcesArgs,
io: KtxCliIo,
@ -1477,59 +1849,27 @@ export async function runKtxSetupSourcesStep(
returnToSourceSelection = true;
break;
}
const connectionId =
sourceChoice.kind === 'existing'
? sourceChoice.connectionId
: (sourceChoice.args.sourceConnectionId ?? `${source}-main`);
const connection =
sourceChoice.kind === 'existing' ? sourceChoice.connection : buildConnection(source, sourceChoice.args);
const rollback =
sourceChoice.kind === 'existing'
? undefined
: await writeSourceConnection(args.projectDir, connectionId, connection);
const validation = await validateSource(source, { projectDir: args.projectDir, connectionId, connection }, deps);
if (!validation.ok) {
await rollback?.();
io.stderr.write(`${validation.message}\n`);
const choiceResult = await saveValidateAndMaybeBuildSource({
args,
source,
sourceChoice,
prompts,
io,
deps,
});
if (choiceResult.status === 'failed') {
return { status: 'failed', projectDir: args.projectDir };
}
if (source === 'metabase' || source === 'looker') {
prompts.log?.(`Validating ${sourceLabel(source)} mapping…`);
const mappingCode = await (deps.runMapping ?? defaultRunMapping)(
args.projectDir,
connectionId,
createSetupPrefixedIo(io),
);
if (mappingCode !== 0) {
await rollback?.();
return { status: 'failed', projectDir: args.projectDir };
if (choiceResult.status === 'back') {
if (args.source) {
return { status: 'back', projectDir: args.projectDir };
}
returnToSourceSelection = true;
break;
}
if (args.runInitialSourceIngest) {
const ingestResult = await runInitialSourceIngestWithRecovery({
args,
connectionId,
io,
prompts,
deps,
});
if (ingestResult === 'failed') {
await rollback?.();
return { status: 'failed', projectDir: args.projectDir };
}
if (ingestResult === 'back') {
await rollback?.();
if (args.source) {
return { status: 'back', projectDir: args.projectDir };
}
returnToSourceSelection = true;
break;
}
} else {
io.stdout.write(`│ Context source ${connectionId} saved. It will be built during the context build step.\n`);
if (!readyConnectionIds.includes(choiceResult.connectionId)) {
readyConnectionIds.push(choiceResult.connectionId);
}
readyConnectionIds.push(connectionId);
}
if (returnToSourceSelection) {
@ -1537,14 +1877,66 @@ export async function runKtxSetupSourcesStep(
}
if (readyConnectionIds.length > 0 && !args.source && args.inputMode !== 'disabled') {
const addMore = await prompts.select({
message: `${readyConnectionIds.length} context source${readyConnectionIds.length > 1 ? 's' : ''} configured (${readyConnectionIds.join(', ')}). Add another?`,
options: [
{ value: 'done', label: 'Done — continue to context build' },
{ value: 'add', label: 'Add another context source' },
],
});
if (addMore === 'add') {
let restartSourceSelection = false;
while (true) {
const addMore = await prompts.select({
message: `${readyConnectionIds.length} context source${readyConnectionIds.length > 1 ? 's' : ''} configured (${readyConnectionIds.join(', ')}). Add another?`,
options: [
{ value: 'done', label: 'Done — continue to context build' },
{ value: 'edit', label: 'Edit an existing context source' },
{ value: 'add', label: 'Add another context source' },
],
});
if (addMore === 'add') {
restartSourceSelection = true;
break;
}
if (addMore === 'edit') {
const editTarget = await chooseContextSourceToEdit({ projectDir: args.projectDir, prompts });
if (editTarget === 'back') {
continue;
}
const projectForEdit = await loadKtxProject({ projectDir: args.projectDir });
const connection = projectForEdit.config.connections[editTarget.connectionId];
if (!connection) {
continue;
}
const sourceChoice = await promptEditedSourceConnection({
args,
source: editTarget.source,
connectionId: editTarget.connectionId,
connection,
prompts,
io,
testGitRepo: deps.testGitRepo,
pickNotionRootPages: deps.pickNotionRootPages,
discoverMetabaseDatabases: deps.discoverMetabaseDatabases,
});
if (sourceChoice === 'back') {
continue;
}
const choiceResult = await saveValidateAndMaybeBuildSource({
args,
source: editTarget.source,
sourceChoice,
prompts,
io,
deps,
});
if (choiceResult.status === 'failed') {
return { status: 'failed', projectDir: args.projectDir };
}
if (choiceResult.status === 'back') {
continue;
}
if (!readyConnectionIds.includes(choiceResult.connectionId)) {
readyConnectionIds.push(choiceResult.connectionId);
}
continue;
}
break;
}
if (restartSourceSelection) {
continue;
}
}

3
packages/cli/src/setup.test.ts
View file

@ -170,7 +170,6 @@ describe('setup status', () => {
' warehouse:',
' driver: postgres',
' url: env:DATABASE_URL',
' readonly: true',
'',
].join('\n'),
'utf-8',
@ -192,7 +191,6 @@ describe('setup status', () => {
' warehouse:',
' driver: postgres',
' url: env:DATABASE_URL',
' readonly: true',
'',
].join('\n'),
'utf-8',
@ -1385,7 +1383,6 @@ describe('setup status', () => {
' warehouse:',
' driver: postgres',
' url: env:DEMO_DATABASE_URL',
' readonly: true',
'',
].join('\n'),
'utf-8',

11
packages/cli/src/sl.test.ts
View file

@ -190,7 +190,7 @@ joins: []
it('runs sl query and prints SQL output', async () => {
const projectDir = join(tempDir, 'project');
const project = await initKtxProject({ projectDir, projectName: 'warehouse' });
project.config.connections.warehouse = { driver: 'postgres', readonly: true };
project.config.connections.warehouse = { driver: 'postgres' };
await project.fileStore.writeFile(
'semantic-layer/warehouse/orders.yaml',
`name: orders
@ -247,7 +247,7 @@ joins: []
it('runs sl query from a JSON query file', async () => {
const projectDir = join(tempDir, 'project');
const project = await initKtxProject({ projectDir, projectName: 'warehouse' });
project.config.connections.warehouse = { driver: 'postgres', readonly: true };
project.config.connections.warehouse = { driver: 'postgres' };
await project.fileStore.writeFile(
'semantic-layer/warehouse/orders.yaml',
`name: orders
@ -314,7 +314,7 @@ joins: []
it('creates default sl query compute through the managed runtime helper', async () => {
const projectDir = join(tempDir, 'project');
const project = await initKtxProject({ projectDir, projectName: 'warehouse' });
project.config.connections.warehouse = { driver: 'postgres', readonly: true };
project.config.connections.warehouse = { driver: 'postgres' };
await project.fileStore.writeFile(
'semantic-layer/warehouse/orders.yaml',
`name: orders
@ -375,7 +375,7 @@ joins: []
it('executes sl query through the injected query executor', async () => {
const projectDir = join(tempDir, 'project');
const project = await initKtxProject({ projectDir, projectName: 'warehouse' });
project.config.connections.warehouse = { driver: 'postgres', url: 'postgres://example/db', readonly: true };
project.config.connections.warehouse = { driver: 'postgres', url: 'postgres://example/db' };
await project.fileStore.writeFile(
'semantic-layer/warehouse/orders.yaml',
`name: orders
@ -471,7 +471,7 @@ joins: []
`);
db.close();
project.config.connections.warehouse = { driver: 'sqlite', path: 'warehouse.db', readonly: true };
project.config.connections.warehouse = { driver: 'sqlite', path: 'warehouse.db' };
await writeFile(
join(projectDir, 'ktx.yaml'),
[
@ -480,7 +480,6 @@ joins: []
' warehouse:',
' driver: sqlite',
' path: warehouse.db',
' readonly: true',
'',
].join('\n'),
'utf-8',

11
packages/cli/src/standalone-smoke.test.ts
View file

@ -84,7 +84,6 @@ async function writeSqliteScanConfig(projectDir: string, dbPath: string, enrich
' warehouse:',
' driver: sqlite',
` path: ${JSON.stringify(dbPath)}`,
' readonly: true',
'ingest:',
' adapters:',
' - live-database',
@ -108,6 +107,10 @@ function expectProjectStderr(result: CliResult, projectDir: string): void {
expect(result).toMatchObject({ code: 0, stderr: `Project: ${projectDir}\n` });
}
function expectSetupStderr(result: CliResult): void {
expect(result).toMatchObject({ code: 0, stderr: '' });
}
async function runSetupNewProject(projectDir: string): Promise<CliResult> {
return await runBuiltCli([
'setup',
@ -139,7 +142,7 @@ describe('standalone built ktx CLI smoke', () => {
const projectDir = join(tempDir, 'project');
const init = await runSetupNewProject(projectDir);
expectProjectStderr(init, projectDir);
expectSetupStderr(init);
expect(init.stdout).toContain(`Project: ${projectDir}`);
const run = await runBuiltCli([
@ -175,7 +178,7 @@ describe('standalone built ktx CLI smoke', () => {
it('runs fast public database ingest through the built binary with manifest artifacts', async () => {
const projectDir = join(tempDir, 'database-ingest-project');
const init = await runSetupNewProject(projectDir);
expectProjectStderr(init, projectDir);
expectSetupStderr(init);
const dbPath = join(projectDir, 'warehouse.db');
createSqliteWarehouse(dbPath);
@ -254,7 +257,7 @@ describe('standalone built ktx CLI smoke', () => {
it('rejects the removed connection add command through the built binary', async () => {
const projectDir = join(tempDir, 'notion-project');
const init = await runSetupNewProject(projectDir);
expectProjectStderr(init, projectDir);
expectSetupStderr(init);
const add = await runBuiltCli([
'connection',

339
packages/cli/src/text-ingest.test.ts Normal file
View file

@ -0,0 +1,339 @@
import { describe, expect, it, vi } from 'vitest';
import type { MemoryCaptureStatus } from '@ktx/context/memory';
import type { KtxLocalProject } from '@ktx/context/project';
import { runKtxTextIngest, type TextMemoryCapturePort } from './text-ingest.js';
function makeIo(options: { isTTY?: boolean } = {}) {
let stdout = '';
let stderr = '';
return {
io: {
stdout: {
isTTY: options.isTTY,
write: (chunk: string) => {
stdout += chunk;
},
},
stderr: {
write: (chunk: string) => {
stderr += chunk;
},
},
},
stdout: () => stdout,
stderr: () => stderr,
};
}
function fakeCapture(
options: {
failRunIds?: Set<string>;
missingStatusRunIds?: Set<string>;
events?: string[];
} = {},
): TextMemoryCapturePort {
let next = 1;
return {
capture: vi.fn(async () => {
const runId = `run-${next++}`;
options.events?.push(`capture:${runId}`);
return { runId };
}),
waitForRun: vi.fn(async (runId: string) => {
options.events?.push(`wait:${runId}`);
}),
status: vi.fn(async (runId: string) => {
options.events?.push(`status:${runId}`);
if (options.missingStatusRunIds?.has(runId)) {
return null;
}
if (options.failRunIds?.has(runId)) {
return {
runId,
status: 'error',
stage: 'capturing',
done: true,
captured: { wiki: [], sl: [], xrefs: [] },
error: `${runId} failed`,
commitHash: null,
skillsLoaded: [],
signalDetected: false,
} satisfies MemoryCaptureStatus;
}
return {
runId,
status: 'done',
stage: 'capturing',
done: true,
captured: { wiki: [`wiki-${runId}`], sl: [`sl-${runId}`], xrefs: [] },
error: null,
commitHash: `commit-${runId}`,
skillsLoaded: ['wiki_capture', 'sl'],
signalDetected: true,
} satisfies MemoryCaptureStatus;
}),
};
}
function fakeProject(projectDir = '/tmp/project'): KtxLocalProject {
return { projectDir } as KtxLocalProject;
}
describe('runKtxTextIngest', () => {
it('captures repeated inline text sequentially with generated internal chat ids', async () => {
const io = makeIo();
const events: string[] = [];
const capture = fakeCapture({ events });
const createMemoryCapture = vi.fn(() => capture);
await expect(
runKtxTextIngest(
{
projectDir: '/tmp/project',
texts: ['Revenue means gross receipts.', 'Orders are completed purchases.'],
files: [],
userId: 'local-cli',
json: true,
failFast: false,
},
io.io,
{
loadProject: vi.fn(async () => fakeProject()),
createMemoryCapture,
now: () => 1_700_000_000_000,
},
),
).resolves.toBe(0);
expect(createMemoryCapture).toHaveBeenCalledWith({ projectDir: '/tmp/project' });
expect(capture.capture).toHaveBeenNthCalledWith(
1,
expect.objectContaining({
userId: 'local-cli',
chatId: 'cli-text-ingest-1700000000000-1',
userMessage: 'Ingest external text artifact "Revenue means gross receipts." into KTX memory.',
assistantMessage: 'Revenue means gross receipts.',
sourceType: 'external_ingest',
}),
);
expect(capture.capture).toHaveBeenNthCalledWith(
2,
expect.objectContaining({
chatId: 'cli-text-ingest-1700000000000-2',
userMessage: 'Ingest external text artifact "Orders are completed purchases." into KTX memory.',
assistantMessage: 'Orders are completed purchases.',
}),
);
expect(capture.capture).not.toHaveBeenCalledWith(expect.objectContaining({ connectionId: expect.anything() }));
expect(events).toEqual(['capture:run-1', 'wait:run-1', 'status:run-1', 'capture:run-2', 'wait:run-2', 'status:run-2']);
expect(JSON.parse(io.stdout())).toMatchObject({
status: 'done',
results: [
{
label: '"Revenue means gross receipts."',
runId: 'run-1',
status: 'done',
captured: { wiki: ['wiki-run-1'], sl: ['sl-run-1'] },
},
{
label: '"Orders are completed purchases."',
runId: 'run-2',
status: 'done',
captured: { wiki: ['wiki-run-2'], sl: ['sl-run-2'] },
},
],
});
});
it('loads files and stdin as batch items and passes a global connection id', async () => {
const io = makeIo();
const capture = fakeCapture();
await expect(
runKtxTextIngest(
{
projectDir: '/tmp/project',
texts: [],
files: ['/tmp/docs/revenue.md', '-'],
connectionId: 'warehouse',
userId: 'agent',
json: false,
failFast: false,
},
io.io,
{
loadProject: vi.fn(async () => fakeProject()),
createMemoryCapture: vi.fn(() => capture),
readFile: vi.fn(async (path) => `file:${path}`),
readStdin: vi.fn(async () => 'stdin content'),
now: () => 10,
},
),
).resolves.toBe(0);
expect(capture.capture).toHaveBeenNthCalledWith(
1,
expect.objectContaining({
connectionId: 'warehouse',
userId: 'agent',
userMessage: 'Ingest external text artifact "revenue.md" into KTX memory.',
assistantMessage: 'file:/tmp/docs/revenue.md',
}),
);
expect(capture.capture).toHaveBeenNthCalledWith(
2,
expect.objectContaining({
connectionId: 'warehouse',
userMessage: 'Ingest external text artifact "stdin" into KTX memory.',
assistantMessage: 'stdin content',
}),
);
expect(io.stdout()).toContain('Ingesting text memory');
expect(io.stdout()).toContain('Texts:');
expect(io.stdout()).toContain('revenue.md');
expect(io.stdout()).toContain('stdin');
});
it('uses bounded inline text previews as labels in plain output and capture metadata', async () => {
const io = makeIo();
const capture = fakeCapture();
const longText = `This inline note is intentionally long ${'x'.repeat(120)}`;
await expect(
runKtxTextIngest(
{
projectDir: '/tmp/project',
texts: ['remember to call me Andrey', ' first line\n\tsecond line ', longText],
files: [],
userId: 'local-cli',
json: false,
failFast: false,
},
io.io,
{
loadProject: vi.fn(async () => fakeProject()),
createMemoryCapture: vi.fn(() => capture),
now: () => 10,
},
),
).resolves.toBe(0);
const output = io.stdout();
expect(output).toContain('"remember to call me Andrey"');
expect(output).toContain('"first line second line"');
expect(output).toContain('"This inline note is intentionally long xxxxxxxx..."');
expect(output).not.toContain('text-1');
expect(output).not.toContain(longText);
expect(capture.capture).toHaveBeenNthCalledWith(
1,
expect.objectContaining({
userMessage: 'Ingest external text artifact "remember to call me Andrey" into KTX memory.',
}),
);
expect(capture.capture).toHaveBeenNthCalledWith(
2,
expect.objectContaining({
userMessage: 'Ingest external text artifact "first line second line" into KTX memory.',
}),
);
expect(capture.capture).toHaveBeenNthCalledWith(
3,
expect.objectContaining({
userMessage: 'Ingest external text artifact "This inline note is intentionally long xxxxxxxx..." into KTX memory.',
}),
);
});
it('continues after an item failure by default and stops when failFast is set', async () => {
const continueIo = makeIo();
const continueCapture = fakeCapture({ failRunIds: new Set(['run-1']) });
await expect(
runKtxTextIngest(
{
projectDir: '/tmp/project',
texts: ['bad', 'good'],
files: [],
userId: 'local-cli',
json: true,
failFast: false,
},
continueIo.io,
{
loadProject: vi.fn(async () => fakeProject()),
createMemoryCapture: vi.fn(() => continueCapture),
},
),
).resolves.toBe(1);
expect(continueCapture.capture).toHaveBeenCalledTimes(2);
expect(JSON.parse(continueIo.stdout())).toMatchObject({
status: 'failed',
results: [
{ label: '"bad"', status: 'error', error: 'run-1 failed' },
{ label: '"good"', status: 'done' },
],
});
const failFastIo = makeIo();
const failFastCapture = fakeCapture({ failRunIds: new Set(['run-1']) });
await expect(
runKtxTextIngest(
{
projectDir: '/tmp/project',
texts: ['bad', 'skipped'],
files: [],
userId: 'local-cli',
json: true,
failFast: true,
},
failFastIo.io,
{
loadProject: vi.fn(async () => fakeProject()),
createMemoryCapture: vi.fn(() => failFastCapture),
},
),
).resolves.toBe(1);
expect(failFastCapture.capture).toHaveBeenCalledTimes(1);
expect(JSON.parse(failFastIo.stdout()).results).toHaveLength(1);
});
it('rejects empty batches and empty text items', async () => {
const noInputIo = makeIo();
await expect(
runKtxTextIngest(
{
projectDir: '/tmp/project',
texts: [],
files: [],
userId: 'local-cli',
json: false,
failFast: false,
},
noInputIo.io,
{ loadProject: vi.fn(), createMemoryCapture: vi.fn() },
),
).resolves.toBe(1);
expect(noInputIo.stderr()).toContain('Provide at least one text item');
const emptyIo = makeIo();
await expect(
runKtxTextIngest(
{
projectDir: '/tmp/project',
texts: [' '],
files: [],
userId: 'local-cli',
json: false,
failFast: false,
},
emptyIo.io,
{ loadProject: vi.fn(), createMemoryCapture: vi.fn() },
),
).resolves.toBe(1);
expect(emptyIo.stderr()).toContain('Text item "text-1" is empty');
});
});

354
packages/cli/src/text-ingest.ts Normal file
View file

@ -0,0 +1,354 @@
import { readFile as fsReadFile } from 'node:fs/promises';
import { basename, resolve } from 'node:path';
import { createLocalProjectMemoryCapture, type MemoryAgentInput, type MemoryCaptureStatus } from '@ktx/context/memory';
import { loadKtxProject, type KtxLocalProject } from '@ktx/context/project';
import type { KtxCliIo } from './cli-runtime.js';
import { createRepainter, initViewState, renderContextBuildView, type ContextBuildTargetState } from './context-build-view.js';
import { formatDuration } from './demo-metrics.js';
import type { KtxPublicIngestPlanTarget } from './public-ingest.js';
export interface KtxTextIngestArgs {
projectDir: string;
texts: string[];
files: string[];
connectionId?: string;
userId: string;
json: boolean;
failFast: boolean;
}
export interface TextMemoryCapturePort {
capture(input: MemoryAgentInput): Promise<{ runId: string }>;
waitForRun(runId: string): Promise<void>;
status(runId: string): Promise<MemoryCaptureStatus | null>;
}
interface TextIngestItem {
label: string;
content: string;
}
interface TextIngestResult {
label: string;
runId: string | null;
status: 'done' | 'error';
captured: MemoryCaptureStatus['captured'];
commitHash: string | null;
error: string | null;
}
export interface KtxTextIngestDeps {
loadProject?: (options: { projectDir: string }) => Promise<KtxLocalProject>;
createMemoryCapture?: (project: KtxLocalProject) => TextMemoryCapturePort;
readFile?: (path: string) => Promise<string>;
readStdin?: () => Promise<string>;
now?: () => number;
}
const INLINE_TEXT_LABEL_MAX_LENGTH = 50;
const ANSI_ESCAPE_PATTERN = /\x1B\[[0-?]*[ -/]*[@-~]/g;
function defaultCreateMemoryCapture(project: KtxLocalProject): TextMemoryCapturePort {
return createLocalProjectMemoryCapture(project);
}
async function defaultReadStdin(): Promise<string> {
const chunks: string[] = [];
process.stdin.setEncoding('utf-8');
for await (const chunk of process.stdin) {
chunks.push(String(chunk));
}
return chunks.join('');
}
async function defaultReadFile(path: string): Promise<string> {
return await fsReadFile(path, 'utf-8');
}
function emptyCaptured(): MemoryCaptureStatus['captured'] {
return { wiki: [], sl: [], xrefs: [] };
}
function normalizedTextPreview(content: string): string {
return content
.replace(ANSI_ESCAPE_PATTERN, '')
.replace(/[\u0000-\u001f\u007f-\u009f]/g, ' ')
.replace(/\s+/g, ' ')
.trim();
}
function truncateLabel(label: string, maxLength = INLINE_TEXT_LABEL_MAX_LENGTH): string {
const chars = Array.from(label);
if (chars.length <= maxLength) {
return label;
}
return `${chars.slice(0, maxLength - 3).join('').trimEnd()}...`;
}
function quoteInlineTextLabel(label: string): string {
return JSON.stringify(label);
}
function makeUniqueLabel(label: string, usedLabels: Set<string>): string {
if (!usedLabels.has(label)) {
return label;
}
for (let index = 2; ; index++) {
const suffix = ` (${index})`;
const candidate = `${truncateLabel(label, INLINE_TEXT_LABEL_MAX_LENGTH - suffix.length)}${suffix}`;
if (!usedLabels.has(candidate)) {
return candidate;
}
}
}
function textLabel(content: string, index: number, usedLabels: Set<string>): string {
const preview = normalizedTextPreview(content);
const baseLabel = preview.length > 0 ? quoteInlineTextLabel(truncateLabel(preview)) : `text-${index + 1}`;
return makeUniqueLabel(baseLabel, usedLabels);
}
function artifactReference(label: string): string {
return label.startsWith('"') ? label : `"${label}"`;
}
function stdinLabel(items: TextIngestItem[]): string {
if (!items.some((item) => item.label === 'stdin')) {
return 'stdin';
}
return `stdin-${items.filter((item) => item.label.startsWith('stdin')).length + 1}`;
}
async function loadItems(args: KtxTextIngestArgs, deps: KtxTextIngestDeps): Promise<TextIngestItem[]> {
const items: TextIngestItem[] = [];
const usedTextLabels = new Set<string>();
args.texts.forEach((content, index) => {
const label = textLabel(content, index, usedTextLabels);
usedTextLabels.add(label);
items.push({ label, content });
});
const readFile = deps.readFile ?? defaultReadFile;
const readStdin = deps.readStdin ?? defaultReadStdin;
for (const file of args.files) {
if (file === '-') {
items.push({ label: stdinLabel(items), content: await readStdin() });
} else {
const path = resolve(file);
items.push({ label: basename(path), content: await readFile(path) });
}
}
return items;
}
function validateItems(items: TextIngestItem[], io: KtxCliIo): boolean {
if (items.length === 0) {
io.stderr.write('Provide at least one text item with --text, a file path, or - for stdin.\n');
return false;
}
for (const item of items) {
if (item.content.trim().length === 0) {
io.stderr.write(`Text item "${item.label}" is empty.\n`);
return false;
}
}
return true;
}
function makeTarget(label: string): KtxPublicIngestPlanTarget {
return {
connectionId: label,
driver: 'text',
operation: 'source-ingest',
debugCommand: '',
steps: ['memory-update'],
};
}
function allTargets(state: ReturnType<typeof initViewState>): ContextBuildTargetState[] {
return [...state.primarySources, ...state.contextSources];
}
function renderTextIngestView(state: ReturnType<typeof initViewState>, styled: boolean): string {
return renderContextBuildView(state, {
styled,
title: 'Ingesting text memory',
contextGroupLabel: 'Texts',
sourceIngestRunningText: 'capturing...',
completedItemName: { singular: 'text', plural: 'texts' },
});
}
function summarizeCaptured(captured: MemoryCaptureStatus['captured']): string {
const parts = [
`wiki=${captured.wiki.length}`,
`sl=${captured.sl.length}`,
`xrefs=${captured.xrefs.length}`,
];
return parts.join(', ');
}
function resultFromStatus(label: string, status: MemoryCaptureStatus): TextIngestResult {
return {
label,
runId: status.runId,
status: status.status === 'done' ? 'done' : 'error',
captured: status.captured,
commitHash: status.commitHash,
error: status.error,
};
}
function errorResult(label: string, runId: string | null, error: unknown): TextIngestResult {
return {
label,
runId,
status: 'error',
captured: emptyCaptured(),
commitHash: null,
error: error instanceof Error ? error.message : String(error),
};
}
function writeJsonResult(args: KtxTextIngestArgs, results: TextIngestResult[], io: KtxCliIo): void {
io.stdout.write(
`${JSON.stringify(
{
status: results.some((result) => result.status === 'error') ? 'failed' : 'done',
projectDir: args.projectDir,
connectionId: args.connectionId ?? null,
results,
},
null,
2,
)}\n`,
);
}
function writePlainFailures(results: TextIngestResult[], io: KtxCliIo): void {
const failures = results.filter((result) => result.status === 'error');
if (failures.length === 0) {
return;
}
io.stdout.write('\nFailed text items:\n');
for (const result of failures) {
io.stdout.write(` ${result.label}: ${result.error ?? 'failed'}\n`);
}
}
export async function runKtxTextIngest(
args: KtxTextIngestArgs,
io: KtxCliIo,
deps: KtxTextIngestDeps = {},
): Promise<number> {
const items = await loadItems(args, deps);
if (!validateItems(items, io)) {
return 1;
}
const project = await (deps.loadProject ?? loadKtxProject)({ projectDir: args.projectDir });
const memoryCapture = (deps.createMemoryCapture ?? defaultCreateMemoryCapture)(project);
const now = deps.now ?? (() => Date.now());
const batchId = now();
const state = initViewState(items.map((item) => makeTarget(item.label)));
const targets = allTargets(state);
const isTTY = io.stdout.isTTY === true && args.json !== true;
const repainter = isTTY ? createRepainter(io) : null;
const results: TextIngestResult[] = [];
state.startedAt = now();
const paint = () => repainter?.paint(renderTextIngestView(state, true));
paint();
let spinnerInterval: ReturnType<typeof setInterval> | null = null;
if (repainter) {
spinnerInterval = setInterval(() => {
const current = now();
state.frame++;
state.totalElapsedMs = state.startedAt === null ? 0 : current - state.startedAt;
for (const target of targets) {
if (target.status === 'running' && target.startedAt !== null) {
target.elapsedMs = current - target.startedAt;
}
}
paint();
}, 140);
}
try {
for (let index = 0; index < items.length; index++) {
const item = items[index]!;
const target = targets[index]!;
target.status = 'running';
target.startedAt = now();
target.detailLine = 'capturing...';
target.progressUpdatedAtMs = target.startedAt;
paint();
let runId: string | null = null;
let result: TextIngestResult;
try {
const captureInput: MemoryAgentInput = {
userId: args.userId,
chatId: `cli-text-ingest-${batchId}-${index + 1}`,
userMessage: `Ingest external text artifact ${artifactReference(item.label)} into KTX memory.`,
assistantMessage: item.content.trim(),
...(args.connectionId ? { connectionId: args.connectionId } : {}),
sourceType: 'external_ingest',
};
const capture = await memoryCapture.capture(captureInput);
runId = capture.runId;
await memoryCapture.waitForRun(runId);
const status = await memoryCapture.status(runId);
if (!status) {
throw new Error(`Memory capture run "${runId}" was not found.`);
}
result = resultFromStatus(item.label, status);
} catch (error) {
result = errorResult(item.label, runId, error);
}
results.push(result);
target.elapsedMs = now() - (target.startedAt ?? now());
target.detailLine = null;
target.status = result.status === 'done' ? 'done' : 'failed';
target.summaryText = result.status === 'done' ? summarizeCaptured(result.captured) : null;
target.failureText = result.status === 'error' ? result.error : null;
paint();
if (result.status === 'error' && args.failFast) {
break;
}
}
} finally {
if (spinnerInterval) {
clearInterval(spinnerInterval);
}
}
if (state.startedAt !== null) {
state.totalElapsedMs = now() - state.startedAt;
}
if (args.json) {
writeJsonResult(args, results, io);
} else if (repainter) {
repainter.paint(renderTextIngestView(state, true));
writePlainFailures(results, io);
} else {
io.stdout.write(renderTextIngestView(state, false));
writePlainFailures(results, io);
}
if (!args.json && results.length > 0) {
const duration = state.totalElapsedMs > 0 ? ` in ${formatDuration(state.totalElapsedMs)}` : '';
const outcome = results.some((result) => result.status === 'error') ? 'finished with failures' : 'finished';
io.stdout.write(`Text memory ingest ${outcome}${duration}.\n`);
}
return results.some((result) => result.status === 'error') ? 1 : 0;
}

7
packages/connector-bigquery/src/connector.test.ts
View file

@ -100,7 +100,6 @@ const connection = {
dataset_id: 'analytics',
credentials_json: JSON.stringify({ project_id: 'project-1', client_email: 'reader@example.test' }),
location: 'US',
readonly: true,
};
describe('KtxBigQueryScanConnector', () => {
@ -112,12 +111,6 @@ describe('KtxBigQueryScanConnector', () => {
datasetIds: ['analytics'],
location: 'US',
});
expect(() =>
bigQueryConnectionConfigFromConfig({
connectionId: 'warehouse',
connection: { ...connection, readonly: false },
}),
).toThrow('Native BigQuery connector requires connections.warehouse.readonly: true');
});
it('introspects datasets, table metadata, primary keys, and normalized types', async () => {

11
packages/connector-bigquery/src/connector.ts
View file

@ -30,7 +30,6 @@ export interface KtxBigQueryConnectionConfig {
dataset_ids?: string[];
credentials_json?: string;
location?: string;
readonly?: boolean;
[key: string]: unknown;
}
@ -194,7 +193,9 @@ function normalizeValue(value: unknown): unknown {
return value;
}
export function isKtxBigQueryConnectionConfig(connection: KtxBigQueryConnectionConfig | undefined): boolean {
export function isKtxBigQueryConnectionConfig(
connection: KtxBigQueryConnectionConfig | undefined,
): connection is KtxBigQueryConnectionConfig {
return String(connection?.driver ?? '').toLowerCase() === 'bigquery';
}
@ -203,11 +204,9 @@ export function bigQueryConnectionConfigFromConfig(input: {
connection: KtxBigQueryConnectionConfig | undefined;
env?: NodeJS.ProcessEnv;
}): KtxBigQueryResolvedConnectionConfig {
const inputDriver = input.connection?.driver ?? 'unknown';
if (!isKtxBigQueryConnectionConfig(input.connection)) {
throw new Error(`Native BigQuery connector cannot run driver "${input.connection?.driver ?? 'unknown'}"`);
}
if (input.connection?.readonly !== true) {
throw new Error(`Native BigQuery connector requires connections.${input.connectionId}.readonly: true`);
throw new Error(`Native BigQuery connector cannot run driver "${inputDriver}"`);
}
const env = input.env ?? process.env;

10
packages/connector-clickhouse/src/connector.test.ts
View file

@ -112,7 +112,6 @@ describe('KtxClickHouseScanConnector', () => {
username: 'reader',
password: 'test-pass', // pragma: allowlist secret
ssl: true,
readonly: true,
},
}),
).toMatchObject({
@ -123,12 +122,6 @@ describe('KtxClickHouseScanConnector', () => {
password: 'test-pass', // pragma: allowlist secret
ssl: true,
});
expect(() =>
clickHouseClientConfigFromConfig({
connectionId: 'warehouse',
connection: { driver: 'clickhouse', host: 'ch.example.test', database: 'analytics', readonly: false },
}),
).toThrow('Native ClickHouse connector requires connections.warehouse.readonly: true');
});
it('introspects schema, primary keys, comments, row counts, and views', async () => {
@ -140,7 +133,6 @@ describe('KtxClickHouseScanConnector', () => {
database: 'analytics',
username: 'reader',
password: 'test-pass', // pragma: allowlist secret
readonly: true,
},
clientFactory: fakeClientFactory(),
now: () => new Date('2026-04-29T14:00:00.000Z'),
@ -189,7 +181,6 @@ describe('KtxClickHouseScanConnector', () => {
database: 'analytics',
username: 'reader',
password: 'test-pass', // pragma: allowlist secret
readonly: true,
},
clientFactory,
});
@ -253,7 +244,6 @@ describe('KtxClickHouseScanConnector', () => {
database: 'analytics',
username: 'reader',
password: 'test-pass', // pragma: allowlist secret
readonly: true,
},
},
clientFactory: fakeClientFactory(),

11
packages/connector-clickhouse/src/connector.ts
View file

@ -35,7 +35,6 @@ export interface KtxClickHouseConnectionConfig {
password?: string;
url?: string;
ssl?: boolean;
readonly?: boolean;
[key: string]: unknown;
}
@ -193,7 +192,9 @@ function isNullableClickHouseType(type: string): boolean {
return type.startsWith('Nullable(') || type.startsWith('LowCardinality(Nullable(');
}
export function isKtxClickHouseConnectionConfig(connection: KtxClickHouseConnectionConfig | undefined): boolean {
export function isKtxClickHouseConnectionConfig(
connection: KtxClickHouseConnectionConfig | undefined,
): connection is KtxClickHouseConnectionConfig {
return String(connection?.driver ?? '').toLowerCase() === 'clickhouse';
}
@ -202,11 +203,9 @@ export function clickHouseClientConfigFromConfig(input: {
connection: KtxClickHouseConnectionConfig | undefined;
env?: NodeJS.ProcessEnv;
}): KtxClickHouseResolvedClientConfig {
const inputDriver = input.connection?.driver ?? 'unknown';
if (!isKtxClickHouseConnectionConfig(input.connection)) {
throw new Error(`Native ClickHouse connector cannot run driver "${input.connection?.driver ?? 'unknown'}"`);
}
if (input.connection?.readonly !== true) {
throw new Error(`Native ClickHouse connector requires connections.${input.connectionId}.readonly: true`);
throw new Error(`Native ClickHouse connector cannot run driver "${inputDriver}"`);
}
const env = input.env ?? process.env;

12
packages/connector-mysql/src/connector.test.ts
View file

@ -92,7 +92,7 @@ function fakePoolFactory(): KtxMysqlPoolFactory {
describe('KtxMysqlScanConnector', () => {
it('resolves MySQL connection configuration safely', () => {
expect(isKtxMysqlConnectionConfig({ driver: 'mysql', host: 'localhost', database: 'analytics', readonly: true })).toBe(true);
expect(isKtxMysqlConnectionConfig({ driver: 'mysql', host: 'localhost', database: 'analytics' })).toBe(true);
expect(isKtxMysqlConnectionConfig({ driver: 'postgres', host: 'localhost', database: 'analytics' })).toBe(false);
expect(
mysqlConnectionPoolConfigFromConfig({
@ -105,7 +105,6 @@ describe('KtxMysqlScanConnector', () => {
username: 'reader',
password: 'secret', // pragma: allowlist secret
ssl: true,
readonly: true,
},
}),
).toMatchObject({
@ -116,12 +115,6 @@ describe('KtxMysqlScanConnector', () => {
password: 'secret', // pragma: allowlist secret
ssl: { rejectUnauthorized: false },
});
expect(() =>
mysqlConnectionPoolConfigFromConfig({
connectionId: 'warehouse',
connection: { driver: 'mysql', host: 'db.example.test', database: 'analytics', readonly: false },
}),
).toThrow('Native MySQL connector requires connections.warehouse.readonly: true');
});
it('introspects schema, primary keys, comments, row counts, views, and foreign keys', async () => {
@ -133,7 +126,6 @@ describe('KtxMysqlScanConnector', () => {
database: 'analytics',
username: 'reader',
password: 'secret', // pragma: allowlist secret
readonly: true,
},
poolFactory: fakePoolFactory(),
now: () => new Date('2026-04-29T12:00:00.000Z'),
@ -192,7 +184,6 @@ describe('KtxMysqlScanConnector', () => {
database: 'analytics',
username: 'reader',
password: 'secret', // pragma: allowlist secret
readonly: true,
},
poolFactory,
});
@ -249,7 +240,6 @@ describe('KtxMysqlScanConnector', () => {
database: 'analytics',
username: 'reader',
password: 'secret', // pragma: allowlist secret
readonly: true,
},
},
poolFactory: fakePoolFactory(),

11
packages/connector-mysql/src/connector.ts
View file

@ -35,7 +35,6 @@ export interface KtxMysqlConnectionConfig {
password?: string;
url?: string;
ssl?: boolean | { rejectUnauthorized?: boolean };
readonly?: boolean;
[key: string]: unknown;
}
@ -232,7 +231,9 @@ function queryParams(params: Record<string, unknown> | unknown[] | undefined): u
return Array.isArray(params) ? params : Object.values(params);
}
export function isKtxMysqlConnectionConfig(connection: KtxMysqlConnectionConfig | undefined): boolean {
export function isKtxMysqlConnectionConfig(
connection: KtxMysqlConnectionConfig | undefined,
): connection is KtxMysqlConnectionConfig {
return String(connection?.driver ?? '').toLowerCase() === 'mysql';
}
@ -241,11 +242,9 @@ export function mysqlConnectionPoolConfigFromConfig(input: {
connection: KtxMysqlConnectionConfig | undefined;
env?: NodeJS.ProcessEnv;
}): KtxMysqlPoolConfig {
const inputDriver = input.connection?.driver ?? 'unknown';
if (!isKtxMysqlConnectionConfig(input.connection)) {
throw new Error(`Native MySQL connector cannot run driver "${input.connection?.driver ?? 'unknown'}"`);
}
if (input.connection?.readonly !== true) {
throw new Error(`Native MySQL connector requires connections.${input.connectionId}.readonly: true`);
throw new Error(`Native MySQL connector cannot run driver "${inputDriver}"`);
}
const env = input.env ?? process.env;

17
packages/connector-postgres/src/connector.test.ts
View file

@ -102,7 +102,7 @@ function metadataResults(): Map<string, FakeQueryResult> {
describe('KtxPostgresScanConnector', () => {
it('resolves configuration safely', () => {
expect(isKtxPostgresConnectionConfig({ driver: 'postgres', url: 'env:DATABASE_URL', readonly: true })).toBe(true);
expect(isKtxPostgresConnectionConfig({ driver: 'postgres', url: 'env:DATABASE_URL' })).toBe(true);
expect(isKtxPostgresConnectionConfig({ driver: 'postgresql', host: 'db', database: 'analytics' })).toBe(true);
expect(isKtxPostgresConnectionConfig({ driver: 'mysql', host: 'db' })).toBe(false);
expect(
@ -115,7 +115,6 @@ describe('KtxPostgresScanConnector', () => {
username: 'reader',
password: 'test-password', // pragma: allowlist secret
schemas: ['analytics', 'public'],
readonly: true,
ssl: true,
rejectUnauthorized: false,
},
@ -134,7 +133,6 @@ describe('KtxPostgresScanConnector', () => {
connection: {
driver: 'postgres',
url: 'env:DEMO_DATABASE_URL',
readonly: true,
},
env: {
DEMO_DATABASE_URL: 'postgresql://reader@demo.example.test:5432/demo?sslmode=prefer',
@ -148,12 +146,16 @@ describe('KtxPostgresScanConnector', () => {
});
expect(libpqPreferConfig).not.toHaveProperty('connectionString');
expect(libpqPreferConfig).not.toHaveProperty('ssl');
expect(() =>
expect(
postgresPoolConfigFromConfig({
connectionId: 'warehouse',
connection: { driver: 'postgres', host: 'db.example.test', database: 'analytics', username: 'reader' },
}),
).toThrow('Native PostgreSQL connector requires connections.warehouse.readonly: true');
).toMatchObject({
host: 'db.example.test',
database: 'analytics',
user: 'reader',
});
});
it('introspects schemas, tables, views, primary keys, comments, row counts, and foreign keys', async () => {
@ -166,7 +168,6 @@ describe('KtxPostgresScanConnector', () => {
username: 'reader',
password: 'test-password', // pragma: allowlist secret
schema: 'public',
readonly: true,
},
poolFactory: fakePoolFactory(metadataResults()),
now: () => new Date('2026-04-29T10:00:00.000Z'),
@ -225,7 +226,6 @@ describe('KtxPostgresScanConnector', () => {
username: 'reader',
password: 'test-password', // pragma: allowlist secret
schema: 'public',
readonly: true,
},
poolFactory: fakePoolFactory(metadataResults()),
});
@ -274,7 +274,6 @@ describe('KtxPostgresScanConnector', () => {
username: 'reader',
password: 'test-password', // pragma: allowlist secret
schema: 'public',
readonly: true,
},
},
poolFactory: fakePoolFactory(metadataResults()),
@ -347,7 +346,6 @@ describe('KtxPostgresScanConnector', () => {
username: 'reader',
password: 'test-password', // pragma: allowlist secret
schema: 'public',
readonly: true,
},
},
poolFactory: endAwarePoolFactory,
@ -383,7 +381,6 @@ describe('KtxPostgresScanConnector', () => {
database: 'analytics',
username: 'reader',
password: 'test-password', // pragma: allowlist secret
readonly: true,
},
poolFactory,
});

11
packages/connector-postgres/src/connector.ts
View file

@ -61,7 +61,6 @@ export interface KtxPostgresConnectionConfig {
sslmode?: string;
sslMode?: string;
rejectUnauthorized?: boolean;
readonly?: boolean;
[key: string]: unknown;
}
@ -291,7 +290,9 @@ function searchPathSchemasFromConnection(connection: KtxPostgresConnectionConfig
return schemas.includes('public') ? schemas : [...schemas, 'public'];
}
export function isKtxPostgresConnectionConfig(connection: KtxPostgresConnectionConfig | undefined): boolean {
export function isKtxPostgresConnectionConfig(
connection: KtxPostgresConnectionConfig | undefined,
): connection is KtxPostgresConnectionConfig {
const driver = String(connection?.driver ?? '').toLowerCase();
return driver === 'postgres' || driver === 'postgresql';
}
@ -301,11 +302,9 @@ export function postgresPoolConfigFromConfig(input: {
connection: KtxPostgresConnectionConfig | undefined;
env?: NodeJS.ProcessEnv;
}): KtxPostgresPoolConfig {
const inputDriver = input.connection?.driver ?? 'unknown';
if (!isKtxPostgresConnectionConfig(input.connection)) {
throw new Error(`Native PostgreSQL connector cannot run driver "${input.connection?.driver ?? 'unknown'}"`);
}
if (input.connection?.readonly !== true) {
throw new Error(`Native PostgreSQL connector requires connections.${input.connectionId}.readonly: true`);
throw new Error(`Native PostgreSQL connector cannot run driver "${inputDriver}"`);
}
const env = input.env ?? process.env;

1
packages/connector-postgres/src/historic-sql-query-client.test.ts
View file

@ -30,7 +30,6 @@ describe('KtxPostgresHistoricSqlQueryClient', () => {
connectionId: 'warehouse',
connection: {
driver: 'postgres',
readonly: true,
url: 'postgresql://readonly:secret@pg.example.test/warehouse', // pragma: allowlist secret
},
poolFactory,

11
packages/connector-snowflake/src/connector.test.ts
View file

@ -78,7 +78,6 @@ describe('KtxSnowflakeScanConnector', () => {
warehouse: 'WH',
database: 'ANALYTICS',
username: 'reader',
readonly: true,
}),
).toBe(true);
expect(isKtxSnowflakeConnectionConfig({ driver: 'bigquery' })).toBe(false);
@ -94,7 +93,6 @@ describe('KtxSnowflakeScanConnector', () => {
schema_name: 'PUBLIC',
username: 'reader',
password: 'fixture-pass', // pragma: allowlist secret
readonly: true,
},
}),
).toMatchObject({
@ -105,12 +103,6 @@ describe('KtxSnowflakeScanConnector', () => {
username: 'reader',
authMethod: 'password',
});
expect(() =>
snowflakeConnectionConfigFromConfig({
connectionId: 'warehouse',
connection: { driver: 'snowflake', account: 'acct', readonly: false },
}),
).toThrow('Native Snowflake connector requires connections.warehouse.readonly: true');
});
it('introspects schema, primary keys, comments, row counts, and dimensions', async () => {
@ -125,7 +117,6 @@ describe('KtxSnowflakeScanConnector', () => {
schema_name: 'PUBLIC',
username: 'reader',
password: 'fixture-pass', // pragma: allowlist secret
readonly: true,
},
driverFactory: fakeDriverFactory(),
now: () => new Date('2026-04-29T18:00:00.000Z'),
@ -185,7 +176,6 @@ describe('KtxSnowflakeScanConnector', () => {
schema_name: 'PUBLIC',
username: 'reader',
password: 'fixture-pass', // pragma: allowlist secret
readonly: true,
},
driverFactory,
});
@ -243,7 +233,6 @@ describe('KtxSnowflakeScanConnector', () => {
schema_name: 'PUBLIC',
username: 'reader',
password: 'fixture-pass', // pragma: allowlist secret
readonly: true,
},
},
driverFactory: fakeDriverFactory(),

13
packages/connector-snowflake/src/connector.ts
View file

@ -38,7 +38,6 @@ export interface KtxSnowflakeConnectionConfig {
privateKey?: string;
passphrase?: string;
role?: string;
readonly?: boolean;
[key: string]: unknown;
}
@ -191,7 +190,9 @@ function toSnowflakeBinds(params: unknown[] | undefined): snowflake.Binds | unde
return params?.map((value) => toSnowflakeBind(value));
}
export function isKtxSnowflakeConnectionConfig(connection: KtxSnowflakeConnectionConfig | undefined): boolean {
export function isKtxSnowflakeConnectionConfig(
connection: KtxSnowflakeConnectionConfig | undefined,
): connection is KtxSnowflakeConnectionConfig {
return String(connection?.driver ?? '').toLowerCase() === 'snowflake';
}
@ -200,11 +201,9 @@ export function snowflakeConnectionConfigFromConfig(input: {
connection: KtxSnowflakeConnectionConfig | undefined;
env?: NodeJS.ProcessEnv;
}): KtxSnowflakeResolvedConnectionConfig {
const inputDriver = input.connection?.driver ?? 'unknown';
if (!isKtxSnowflakeConnectionConfig(input.connection)) {
throw new Error(`Native Snowflake connector cannot run driver "${input.connection?.driver ?? 'unknown'}"`);
}
if (input.connection?.readonly !== true) {
throw new Error(`Native Snowflake connector requires connections.${input.connectionId}.readonly: true`);
throw new Error(`Native Snowflake connector cannot run driver "${inputDriver}"`);
}
const env = input.env ?? process.env;
const authMethod = input.connection?.authMethod ?? 'password';
@ -395,7 +394,7 @@ class SnowflakeSdkDriver implements KtxSnowflakeDriver {
private async createConnection(): Promise<snowflake.Connection> {
const patch = await this.sdkOptionsProvider?.resolve({
account: this.resolved.account,
connection: { ...this.resolved, driver: 'snowflake', readonly: true },
connection: { ...this.resolved, driver: 'snowflake' },
});
if (patch?.close) {
this.closeSdkOptions.push(patch.close);

26
packages/connector-sqlite/src/connector.test.ts
View file

@ -53,45 +53,43 @@ describe('KtxSqliteScanConnector', () => {
writeFileSync(pointerPath, dbPath, 'utf-8');
try {
expect(isKtxSqliteConnectionConfig({ driver: 'sqlite', path: 'warehouse.db', readonly: true })).toBe(true);
expect(isKtxSqliteConnectionConfig({ driver: 'postgres', url: 'env:DATABASE_URL', readonly: true })).toBe(
false,
);
expect(isKtxSqliteConnectionConfig({ driver: 'sqlite', path: 'warehouse.db' })).toBe(true);
expect(isKtxSqliteConnectionConfig({ driver: 'postgres', url: 'env:DATABASE_URL' })).toBe(false);
expect(
sqliteDatabasePathFromConfig({
connectionId: 'warehouse',
projectDir: tempDir,
connection: { driver: 'sqlite', path: 'warehouse.db', readonly: true },
connection: { driver: 'sqlite', path: 'warehouse.db' },
}),
).toBe(dbPath);
expect(
sqliteDatabasePathFromConfig({
connectionId: 'warehouse',
projectDir: tempDir,
connection: { driver: 'sqlite', url: 'env:KTX_SQLITE_TEST_URL', readonly: true },
connection: { driver: 'sqlite', url: 'env:KTX_SQLITE_TEST_URL' },
}),
).toBe(dbPath);
expect(
sqliteDatabasePathFromConfig({
connectionId: 'warehouse',
projectDir: tempDir,
connection: { driver: 'sqlite', url: `file://${dbPath}`, readonly: true },
connection: { driver: 'sqlite', url: `file://${dbPath}` },
}),
).toBe(dbPath);
expect(
sqliteDatabasePathFromConfig({
connectionId: 'warehouse',
projectDir: tempDir,
connection: { driver: 'sqlite', path: `file:${pointerPath}`, readonly: true },
connection: { driver: 'sqlite', path: `file:${pointerPath}` },
}),
).toBe(dbPath);
expect(() =>
expect(
sqliteDatabasePathFromConfig({
connectionId: 'warehouse',
projectDir: tempDir,
connection: { driver: 'sqlite', path: 'warehouse.db', readonly: false },
connection: { driver: 'sqlite', path: 'warehouse.db' },
}),
).toThrow('Native SQLite connector requires connections.warehouse.readonly: true');
).toBe(dbPath);
} finally {
if (originalDatabaseUrl === undefined) {
delete process.env.KTX_SQLITE_TEST_URL;
@ -104,7 +102,7 @@ describe('KtxSqliteScanConnector', () => {
it('introspects schema, primary keys, row counts, views, and foreign keys', async () => {
const connector = new KtxSqliteScanConnector({
connectionId: 'warehouse',
connection: { driver: 'sqlite', path: dbPath, readonly: true },
connection: { driver: 'sqlite', path: dbPath },
now: () => new Date('2026-04-29T10:00:00.000Z'),
});
@ -151,7 +149,7 @@ describe('KtxSqliteScanConnector', () => {
it('runs samples, distinct values, statistics, and read-only SQL', async () => {
const connector = new KtxSqliteScanConnector({
connectionId: 'warehouse',
connection: { driver: 'sqlite', path: dbPath, readonly: true },
connection: { driver: 'sqlite', path: dbPath },
});
await expect(
@ -199,7 +197,7 @@ describe('KtxSqliteScanConnector', () => {
const introspection = createSqliteLiveDatabaseIntrospection({
projectDir: tempDir,
connections: {
warehouse: { driver: 'sqlite', path: 'warehouse.db', readonly: true },
warehouse: { driver: 'sqlite', path: 'warehouse.db' },
},
now: () => new Date('2026-04-29T10:00:00.000Z'),
});

11
packages/connector-sqlite/src/connector.ts
View file

@ -29,7 +29,6 @@ export interface KtxSqliteConnectionConfig {
path?: string;
url?: string;
file_path?: string;
readonly?: boolean;
[key: string]: unknown;
}
@ -135,17 +134,17 @@ function stripLeadingSqlComments(sql: string): string {
return sql.slice(index);
}
export function isKtxSqliteConnectionConfig(connection: KtxSqliteConnectionConfig | undefined): boolean {
export function isKtxSqliteConnectionConfig(
connection: KtxSqliteConnectionConfig | undefined,
): connection is KtxSqliteConnectionConfig {
const driver = String(connection?.driver ?? '').toLowerCase();
return driver === 'sqlite' || driver === 'sqlite3';
}
export function sqliteDatabasePathFromConfig(input: SqliteDatabasePathInput): string {
const inputDriver = input.connection?.driver ?? 'unknown';
if (!isKtxSqliteConnectionConfig(input.connection)) {
throw new Error(`Native SQLite connector cannot run driver "${input.connection?.driver ?? 'unknown'}"`);
}
if (input.connection?.readonly !== true) {
throw new Error(`Native SQLite connector requires connections.${input.connectionId}.readonly: true`);
throw new Error(`Native SQLite connector cannot run driver "${inputDriver}"`);
}
const configuredPath =
stringConfigValue(input.connection, 'path') ??

11
packages/connector-sqlserver/src/connector.test.ts
View file

@ -145,7 +145,6 @@ describe('KtxSqlServerScanConnector', () => {
driver: 'sqlserver',
host: 'localhost',
database: 'analytics',
readonly: true,
}),
).toBe(true);
expect(isKtxSqlServerConnectionConfig({ driver: 'mysql', host: 'localhost', database: 'analytics' })).toBe(false);
@ -159,7 +158,6 @@ describe('KtxSqlServerScanConnector', () => {
database: 'analytics',
username: 'reader',
trustServerCertificate: false,
readonly: true,
},
}),
).toMatchObject({
@ -169,12 +167,6 @@ describe('KtxSqlServerScanConnector', () => {
user: 'reader',
options: { encrypt: true, trustServerCertificate: false },
});
expect(() =>
sqlServerConnectionPoolConfigFromConfig({
connectionId: 'warehouse',
connection: { driver: 'sqlserver', host: 'db.example.test', database: 'analytics', readonly: false },
}),
).toThrow('Native SQL Server connector requires connections.warehouse.readonly: true');
});
it('introspects schema, primary keys, comments, row counts, views, and foreign keys', async () => {
@ -186,7 +178,6 @@ describe('KtxSqlServerScanConnector', () => {
database: 'analytics',
username: 'reader',
schema: 'dbo',
readonly: true,
},
poolFactory: fakePoolFactory(),
now: () => new Date('2026-04-29T16:00:00.000Z'),
@ -246,7 +237,6 @@ describe('KtxSqlServerScanConnector', () => {
database: 'analytics',
username: 'reader',
schema: 'dbo',
readonly: true,
},
poolFactory,
});
@ -315,7 +305,6 @@ describe('KtxSqlServerScanConnector', () => {
database: 'analytics',
username: 'reader',
schema: 'dbo',
readonly: true,
},
},
poolFactory: fakePoolFactory(),

11
packages/connector-sqlserver/src/connector.ts
View file

@ -37,7 +37,6 @@ export interface KtxSqlServerConnectionConfig {
schema?: string;
schemas?: string[];
trustServerCertificate?: boolean;
readonly?: boolean;
[key: string]: unknown;
}
@ -234,7 +233,9 @@ function limitSqlForSqlServerExecution(sqlText: string, maxRows: number | undefi
return `SELECT TOP ${maxRows} * FROM (${trimmed}) AS ktx_query_result`;
}
export function isKtxSqlServerConnectionConfig(connection: KtxSqlServerConnectionConfig | undefined): boolean {
export function isKtxSqlServerConnectionConfig(
connection: KtxSqlServerConnectionConfig | undefined,
): connection is KtxSqlServerConnectionConfig {
return String(connection?.driver ?? '').toLowerCase() === 'sqlserver';
}
@ -243,11 +244,9 @@ export function sqlServerConnectionPoolConfigFromConfig(input: {
connection: KtxSqlServerConnectionConfig | undefined;
env?: NodeJS.ProcessEnv;
}): KtxSqlServerPoolConfig {
const inputDriver = input.connection?.driver ?? 'unknown';
if (!isKtxSqlServerConnectionConfig(input.connection)) {
throw new Error(`Native SQL Server connector cannot run driver "${input.connection?.driver ?? 'unknown'}"`);
}
if (input.connection?.readonly !== true) {
throw new Error(`Native SQL Server connector requires connections.${input.connectionId}.readonly: true`);
throw new Error(`Native SQL Server connector cannot run driver "${inputDriver}"`);
}
const env = input.env ?? process.env;

6
packages/context/src/connections/local-query-executor.test.ts
View file

@ -26,14 +26,14 @@ describe('createDefaultLocalQueryExecutor', () => {
await expect(
executor.execute({
connectionId: 'pg',
connection: { driver: 'postgres', readonly: true },
connection: { driver: 'postgres' },
sql: 'select 1',
}),
).resolves.toMatchObject({ headers: ['pg'] });
await expect(
executor.execute({
connectionId: 'local',
connection: { driver: 'sqlite', readonly: true },
connection: { driver: 'sqlite' },
sql: 'select 1',
}),
).resolves.toMatchObject({ headers: ['sqlite'] });
@ -51,7 +51,7 @@ describe('createDefaultLocalQueryExecutor', () => {
await expect(
executor.execute({
connectionId: 'warehouse',
connection: { driver: 'snowflake', readonly: true },
connection: { driver: 'snowflake' },
sql: 'select 1',
}),
).rejects.toThrow('No local query executor is configured for driver "snowflake".');

16
packages/context/src/connections/postgres-query-executor.test.ts
View file

@ -37,7 +37,7 @@ describe('createPostgresQueryExecutor', () => {
const result = await executor.execute({
connectionId: 'warehouse',
connection: { driver: 'postgres', url: 'postgres://example/db', readonly: true },
connection: { driver: 'postgres', url: 'postgres://example/db' },
sql: 'select status, count(*) as order_count from public.orders group by status',
maxRows: 50,
});
@ -80,7 +80,7 @@ describe('createPostgresQueryExecutor', () => {
await expect(
executor.execute({
connectionId: 'warehouse',
connection: { driver: 'postgres', url: 'postgres://example/db', readonly: true },
connection: { driver: 'postgres', url: 'postgres://example/db' },
sql: 'select * from broken',
maxRows: 10,
}),
@ -89,23 +89,15 @@ describe('createPostgresQueryExecutor', () => {
expect(client.end).toHaveBeenCalledTimes(1);
});
it('requires a Postgres url and read-only connection config', async () => {
it('requires a Postgres url', async () => {
const executor = createPostgresQueryExecutor({ clientFactory: vi.fn() });
await expect(
executor.execute({
connectionId: 'warehouse',
connection: { driver: 'postgres', readonly: true },
connection: { driver: 'postgres' },
sql: 'select 1',
}),
).rejects.toThrow('Local Postgres execution requires connections.warehouse.url');
await expect(
executor.execute({
connectionId: 'warehouse',
connection: { driver: 'postgres', url: 'postgres://example/db', readonly: false },
sql: 'select 1',
}),
).rejects.toThrow('Local query execution requires connections.warehouse.readonly: true');
});
});

10
packages/context/src/connections/postgres-query-executor.ts
View file

@ -37,18 +37,16 @@ export function createPostgresQueryExecutor(options: PostgresQueryExecutorOption
return {
async execute(input: KtxSqlQueryExecutionInput): Promise<KtxSqlQueryExecutionResult> {
const driver = connectionDriver(input);
const connection = input.connection;
if (driver !== 'postgres' && driver !== 'postgresql') {
throw new Error(`Local Postgres execution cannot run driver "${input.connection?.driver ?? 'unknown'}".`);
throw new Error(`Local Postgres execution cannot run driver "${connection?.driver ?? 'unknown'}".`);
}
if (input.connection?.readonly !== true) {
throw new Error(`Local query execution requires connections.${input.connectionId}.readonly: true.`);
}
if (typeof input.connection.url !== 'string' || input.connection.url.trim().length === 0) {
if (typeof connection?.url !== 'string' || connection.url.trim().length === 0) {
throw new Error(`Local Postgres execution requires connections.${input.connectionId}.url.`);
}
const client = clientFactory({
connectionString: input.connection.url,
connectionString: connection.url,
statement_timeout: options.statementTimeoutMs ?? 30_000,
query_timeout: options.queryTimeoutMs ?? 35_000,
connectionTimeoutMillis: options.connectionTimeoutMs ?? 5_000,

25
packages/context/src/connections/sqlite-query-executor.test.ts
View file

@ -38,7 +38,7 @@ describe('createSqliteQueryExecutor', () => {
const result = await executor.execute({
connectionId: 'warehouse',
projectDir: tempDir,
connection: { driver: 'sqlite', path: 'warehouse.db', readonly: true },
connection: { driver: 'sqlite', path: 'warehouse.db' },
sql: 'select status, count(*) as order_count from orders group by status order by status',
maxRows: 10,
});
@ -60,7 +60,7 @@ describe('createSqliteQueryExecutor', () => {
sqliteDatabasePathFromConnection({
connectionId: 'warehouse',
projectDir: tempDir,
connection: { driver: 'sqlite', url: `file://${dbPath}`, readonly: true },
connection: { driver: 'sqlite', url: `file://${dbPath}` },
sql: 'select 1',
}),
).toBe(dbPath);
@ -74,7 +74,7 @@ describe('createSqliteQueryExecutor', () => {
sqliteDatabasePathFromConnection({
connectionId: 'warehouse',
projectDir: tempDir,
connection: { driver: 'sqlite', path: `file:${pointerPath}`, readonly: true },
connection: { driver: 'sqlite', path: `file:${pointerPath}` },
sql: 'select 1',
}),
).toBe(dbPath);
@ -89,7 +89,7 @@ describe('createSqliteQueryExecutor', () => {
sqliteDatabasePathFromConnection({
connectionId: 'warehouse',
projectDir: tempDir,
connection: { driver: 'sqlite', url: 'env:KTX_SQLITE_TEST_URL', readonly: true },
connection: { driver: 'sqlite', url: 'env:KTX_SQLITE_TEST_URL' },
sql: 'select 1',
}),
).toBe(dbPath);
@ -109,20 +109,20 @@ describe('createSqliteQueryExecutor', () => {
executor.execute({
connectionId: 'warehouse',
projectDir: tempDir,
connection: { driver: 'sqlite', path: 'warehouse.db', readonly: true },
connection: { driver: 'sqlite', path: 'warehouse.db' },
sql: 'delete from orders',
}),
).rejects.toThrow('Only read-only SELECT/WITH queries can be executed locally');
});
it('requires a SQLite driver, read-only config, and a database path', async () => {
it('requires a SQLite driver and a database path', async () => {
const executor = createSqliteQueryExecutor();
await expect(
executor.execute({
connectionId: 'warehouse',
projectDir: tempDir,
connection: { driver: 'postgres', path: 'warehouse.db', readonly: true },
connection: { driver: 'postgres', path: 'warehouse.db' },
sql: 'select 1',
}),
).rejects.toThrow('Local SQLite execution cannot run driver "postgres"');
@ -131,16 +131,7 @@ describe('createSqliteQueryExecutor', () => {
executor.execute({
connectionId: 'warehouse',
projectDir: tempDir,
connection: { driver: 'sqlite', path: 'warehouse.db', readonly: false },
sql: 'select 1',
}),
).rejects.toThrow('Local query execution requires connections.warehouse.readonly: true');
await expect(
executor.execute({
connectionId: 'warehouse',
projectDir: tempDir,
connection: { driver: 'sqlite', readonly: true },
connection: { driver: 'sqlite' },
sql: 'select 1',
}),
).rejects.toThrow('Local SQLite execution requires connections.warehouse.path or connections.warehouse.url');

3
packages/context/src/connections/sqlite-query-executor.ts
View file

@ -54,9 +54,6 @@ export function sqliteDatabasePathFromConnection(input: KtxSqlQueryExecutionInpu
if (driver !== 'sqlite' && driver !== 'sqlite3') {
throw new Error(`Local SQLite execution cannot run driver "${input.connection?.driver ?? 'unknown'}".`);
}
if (input.connection?.readonly !== true) {
throw new Error(`Local query execution requires connections.${input.connectionId}.readonly: true.`);
}
const pathValue = stringConfigValue(input.connection, 'path');
const urlValue = stringConfigValue(input.connection, 'url');

2
packages/context/src/ingest/adapters/historic-sql/redaction.test.ts
View file

@ -9,7 +9,7 @@ describe('historic-SQL redaction', () => {
]);
const sql =
"select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'";
"select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; // pragma: allowlist secret
expect(redactHistoricSqlText(sql, redactors)).toBe(
"select * from public.api_events where api_key = '[REDACTED]' and note = '[REDACTED]'",

2
packages/context/src/ingest/adapters/historic-sql/stage-unified.test.ts
View file

@ -169,7 +169,7 @@ describe('stageHistoricSqlAggregatedSnapshot', () => {
it('redacts configured SQL substrings in staged artifacts while analyzing original SQL', async () => {
const stagedDir = await tempDir();
const originalSql =
"select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'";
"select * from public.api_events where api_key = 'sk_live_abc123' and note = 'Secret_Token_9f'"; // pragma: allowlist secret
const reader: HistoricSqlReader = {
async probe() {
return { warnings: [], info: [] };

9
packages/context/src/ingest/adapters/live-database/daemon-introspection.test.ts
View file

@ -45,7 +45,6 @@ describe('createDaemonLiveDatabaseIntrospection', () => {
warehouse: {
driver: 'postgres',
url: 'postgres://localhost:5432/warehouse',
readonly: true,
},
},
schemas: ['public'],
@ -157,7 +156,6 @@ describe('createDaemonLiveDatabaseIntrospection', () => {
warehouse: {
driver: 'postgresql',
url: 'postgres://localhost:5432/warehouse',
readonly: true,
},
},
baseUrl: `http://127.0.0.1:${address.port}`,
@ -186,20 +184,18 @@ describe('createDaemonLiveDatabaseIntrospection', () => {
}
});
it('requires a configured read-only postgres connection with a url', async () => {
it('requires a configured postgres connection with a url', async () => {
const introspection = createDaemonLiveDatabaseIntrospection({
connections: {
warehouse: {
driver: 'postgres',
url: 'postgres://localhost:5432/warehouse',
readonly: false,
},
},
runJson: vi.fn(async () => daemonResponse),
});
await expect(introspection.extractSchema('warehouse')).rejects.toThrow(
'Local live-database ingest requires connections.warehouse.readonly: true.',
'Local live-database ingest requires connections.warehouse.url.',
);
});
@ -210,7 +206,6 @@ describe('createDaemonLiveDatabaseIntrospection', () => {
warehouse: {
driver: 'snowflake',
url: 'snowflake://example',
readonly: true,
},
},
runJson,

3
packages/context/src/ingest/adapters/live-database/daemon-introspection.ts
View file

@ -162,9 +162,6 @@ function requirePostgresConnection(
if (driver !== 'postgres') {
throw new Error(`Local live-database ingest cannot run driver "${connection?.driver ?? 'unknown'}".`);
}
if (connection?.readonly !== true) {
throw new Error(`Local live-database ingest requires connections.${connectionId}.readonly: true.`);
}
if (typeof connection.url !== 'string' || connection.url.trim().length === 0) {
throw new Error(`Local live-database ingest requires connections.${connectionId}.url.`);
}

2
packages/context/src/ingest/adapters/metabase/client.test.ts
View file

@ -92,7 +92,7 @@ describe('MetabaseClient retry exhaustion', () => {
.mockResolvedValueOnce(new Response(JSON.stringify([]), { status: 200 }));
const client = new MetabaseClient(
{ apiUrl: 'https://metabase.example.test', apiKey: 'key' },
{ apiUrl: 'https://metabase.example.test', apiKey: 'key' }, // pragma: allowlist secret
{
...DEFAULT_METABASE_CLIENT_CONFIG,
baseDelayMs: 0,

2
packages/context/src/ingest/adapters/metabase/local-metabase.adapter.test.ts
View file

@ -39,7 +39,7 @@ describe('metabaseRuntimeConfigFromLocalConnection', () => {
const connection: KtxProjectConnectionConfig = {
driver: 'metabase',
api_url: 'https://metabase.example.com',
api_key_ref: `file:${keyPath}`,
api_key_ref: `file:${keyPath}`, // pragma: allowlist secret
};
expect(metabaseRuntimeConfigFromLocalConnection('prod-metabase', connection)).toEqual({

1
packages/context/src/ingest/local-stage-ingest.test.ts
View file

@ -39,7 +39,6 @@ async function writeLiveDatabaseConfig(projectDir: string): Promise<void> {
' warehouse:',
' driver: postgres',
' url: postgres://localhost:5432/warehouse',
' readonly: true',
'ingest:',
' adapters:',
' - live-database',

7
packages/context/src/mcp/local-project-ports.test.ts
View file

@ -75,7 +75,6 @@ describe('createLocalProjectMcpContextPorts', () => {
project.config.connections.warehouse = {
driver: 'postgres',
url: 'env:DATABASE_URL',
readonly: true,
};
const ports = createLocalProjectMcpContextPorts(project);
@ -89,7 +88,6 @@ describe('createLocalProjectMcpContextPorts', () => {
project.config.connections.warehouse = {
driver: 'postgres',
url: 'env:DATABASE_URL',
readonly: true,
};
const connector = testConnector();
const createConnector = vi.fn(async () => connector);
@ -125,7 +123,6 @@ describe('createLocalProjectMcpContextPorts', () => {
const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' });
project.config.connections.warehouse = {
driver: 'postgres',
readonly: true,
};
project.config.ingest.adapters = ['fake'];
project.config.ingest.embeddings = {
@ -633,7 +630,6 @@ describe('createLocalProjectMcpContextPorts', () => {
project.config.connections.warehouse = {
driver: 'postgres',
url: 'env:DATABASE_URL',
readonly: true,
};
const shapeOnlyPorts = createLocalProjectMcpContextPorts(project);
await shapeOnlyPorts.semanticLayer?.writeSource({
@ -720,7 +716,6 @@ describe('createLocalProjectMcpContextPorts', () => {
project.config.connections.warehouse = {
driver: 'postgres',
url: 'env:DATABASE_URL',
readonly: true,
};
const shapeOnlyPorts = createLocalProjectMcpContextPorts(project);
await shapeOnlyPorts.semanticLayer?.writeSource({
@ -958,7 +953,6 @@ describe('createLocalProjectMcpContextPorts', () => {
project.config.connections.warehouse = {
driver: 'postgres',
url: 'postgres://localhost:5432/warehouse',
readonly: true,
};
project.config.ingest.adapters = ['live-database'];
project.config.llm = {
@ -1034,7 +1028,6 @@ describe('createLocalProjectMcpContextPorts', () => {
project.config.connections.warehouse = {
driver: 'postgres',
url: 'env:DATABASE_URL',
readonly: true,
};
project.config.ingest.adapters = ['live-database'];
const ports = createLocalProjectMcpContextPorts(project, {

2
packages/context/src/memory/local-memory.test.ts
View file

@ -145,7 +145,7 @@ describe('createLocalProjectMemoryCapture', () => {
it('captures a semantic-layer source for a named local connection id', async () => {
const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' });
project.config.connections.warehouse = { driver: 'postgres', readonly: true };
project.config.connections.warehouse = { driver: 'postgres' };
const agentRunner = {
runLoop: async ({
toolSet,

1
packages/context/src/project/config.ts
View file

@ -69,7 +69,6 @@ export interface KtxProjectScanConfig {
export interface KtxProjectConnectionConfig {
driver: string;
url?: string;
readonly?: boolean;
[key: string]: unknown;
}

6
packages/context/src/scan/local-scan.test.ts
View file

@ -110,7 +110,6 @@ async function writeLiveDatabaseConfig(projectDir: string): Promise<void> {
' warehouse:',
' driver: postgres',
' url: env:DATABASE_URL',
' readonly: true',
'ingest:',
' adapters:',
' - live-database',
@ -1043,7 +1042,6 @@ describe('local scan', () => {
' warehouse:',
' driver: postgres',
' url: env:DATABASE_URL',
' readonly: true',
'ingest:',
' adapters:',
' - live-database',
@ -1400,7 +1398,6 @@ describe('local scan', () => {
' warehouse:',
' driver: sqlite',
' path: warehouse.db',
' readonly: true',
'ingest:',
' adapters:',
' - live-database',
@ -1433,7 +1430,6 @@ describe('local scan', () => {
' warehouse:',
' driver: mysql',
' url: env:MYSQL_URL',
' readonly: true',
'ingest:',
' adapters:',
' - live-database',
@ -1469,7 +1465,6 @@ describe('local scan', () => {
' database: analytics',
' username: reader',
' password: env:CLICKHOUSE_PASSWORD',
' readonly: true',
'ingest:',
' adapters:',
' - live-database',
@ -1505,7 +1500,6 @@ describe('local scan', () => {
' database: analytics',
' username: reader',
' schema: dbo',
' readonly: true',
'ingest:',
' adapters:',
' - live-database',

1
packages/context/src/scan/relationship-artifacts.test.ts
View file

@ -24,7 +24,6 @@ async function writeWarehouseConfig(projectDir: string): Promise<void> {
' warehouse:',
' driver: sqlite',
' path: warehouse.db',
' readonly: true',
'ingest:',
' adapters:',
' - live-database',

1
packages/context/src/scan/relationship-review-decisions.test.ts
View file

@ -28,7 +28,6 @@ async function createProject(projectDir: string): Promise<void> {
' warehouse:',
' driver: sqlite',
' path: warehouse.db',
' readonly: true',
'ingest:',
' adapters:',
' - live-database',

6
packages/context/src/sl/local-query.test.ts
View file

@ -14,7 +14,7 @@ describe('compileLocalSlQuery', () => {
beforeEach(async () => {
tempDir = await mkdtemp(join(tmpdir(), 'ktx-local-query-'));
project = await initKtxProject({ projectDir: join(tempDir, 'project'), projectName: 'warehouse' });
project.config.connections.warehouse = { driver: 'postgres', readonly: true };
project.config.connections.warehouse = { driver: 'postgres' };
await project.fileStore.writeFile(
'semantic-layer/warehouse/orders.yaml',
`name: orders
@ -222,7 +222,7 @@ grain: []
expect(queryExecutor.execute).toHaveBeenCalledWith({
connectionId: 'warehouse',
projectDir: project.projectDir,
connection: { driver: 'postgres', readonly: true },
connection: { driver: 'postgres' },
sql: 'select status, count(*) as order_count from public.orders group by status',
maxRows: 10,
});
@ -248,7 +248,7 @@ grain: []
});
it('requires connectionId when multiple connections are configured', async () => {
project.config.connections.analytics = { driver: 'bigquery', readonly: true };
project.config.connections.analytics = { driver: 'bigquery' };
await expect(
compileLocalSlQuery(project, {

10
packages/llm/src/embedding-health.test.ts
View file

@ -17,13 +17,13 @@ describe('KTX embedding health check', () => {
backend: 'openai',
model: 'text-embedding-3-small',
dimensions: 3,
openai: { apiKey: 'sk-openai-test' },
openai: { apiKey: 'sk-openai-test' }, // pragma: allowlist secret
},
{ deps: { createOpenAIClient } },
),
).resolves.toEqual({ ok: true });
expect(createOpenAIClient).toHaveBeenCalledWith({ apiKey: 'sk-openai-test', baseURL: undefined });
expect(createOpenAIClient).toHaveBeenCalledWith({ apiKey: 'sk-openai-test', baseURL: undefined }); // pragma: allowlist secret
});
it('returns failed when the provider returns the wrong dimensions', async () => {
@ -41,7 +41,7 @@ describe('KTX embedding health check', () => {
backend: 'openai',
model: 'text-embedding-3-small',
dimensions: 3,
openai: { apiKey: 'sk-openai-test' },
openai: { apiKey: 'sk-openai-test' }, // pragma: allowlist secret
},
{ deps: { createOpenAIClient } },
),
@ -66,7 +66,7 @@ describe('KTX embedding health check', () => {
backend: 'openai',
model: 'text-embedding-3-small',
dimensions: 3,
openai: { apiKey: 'sk-openai-secret' },
openai: { apiKey: 'sk-openai-secret' }, // pragma: allowlist secret
},
{ deps: { createOpenAIClient } },
),
@ -94,7 +94,7 @@ describe('KTX embedding health check', () => {
backend: 'openai',
model: 'text-embedding-3-small',
dimensions: 3,
openai: { apiKey: 'sk-openai-test' },
openai: { apiKey: 'sk-openai-test' }, // pragma: allowlist secret
},
{ timeoutMs: 1, deps: { createOpenAIClient } },
),

6
packages/llm/src/model-health.test.ts
View file

@ -14,7 +14,7 @@ describe('KTX LLM health check', () => {
runKtxLlmHealthCheck(
{
backend: 'anthropic',
anthropic: { apiKey: 'sk-ant-test' },
anthropic: { apiKey: 'sk-ant-test' }, // pragma: allowlist secret
modelSlots: { default: 'claude-sonnet-4-6' },
},
{ deps: { createAnthropic, generateText, devtoolsEnabled: true, wrapLanguageModel } },
@ -23,7 +23,7 @@ describe('KTX LLM health check', () => {
expect(createAnthropic).toHaveBeenCalledWith(
expect.objectContaining({
apiKey: 'sk-ant-test',
apiKey: 'sk-ant-test', // pragma: allowlist secret
}),
);
expect(generateText).toHaveBeenCalledWith(
@ -46,7 +46,7 @@ describe('KTX LLM health check', () => {
runKtxLlmHealthCheck(
{
backend: 'anthropic',
anthropic: { apiKey: 'sk-ant-secret' },
anthropic: { apiKey: 'sk-ant-secret' }, // pragma: allowlist secret
modelSlots: { default: 'claude-sonnet-4-6' },
},
{

1
pyproject.toml
View file

@ -13,6 +13,7 @@ Issues = "https://github.com/kaelio/ktx/issues"
[dependency-groups]
dev = [
"pre-commit>=4.6.0",
"pytest>=9.0.2",
"ruff>=0.8.4",
]

4
python/ktx-daemon/src/ktx_daemon/sql_analysis.py
View file

@ -130,7 +130,9 @@ def _analyze_one(
)
def _analyze_payload(payload: tuple[str, str, str]) -> tuple[str, AnalyzeSqlBatchResult]:
def _analyze_payload(
payload: tuple[str, str, str],
) -> tuple[str, AnalyzeSqlBatchResult]:
item_id, sql, dialect = payload
return _analyze_one(item_id, sql, dialect)

1
scripts/build-adventureworks-oltp-fixture.mjs
View file

@ -231,7 +231,6 @@ async function main() {
driver: 'sqlserver',
url,
schemas: ['dbo', 'HumanResources', 'Person', 'Production', 'Purchasing', 'Sales'],
readonly: true,
trustServerCertificate: true,
},
now: () => new Date('2026-05-07T00:00:00.000Z'),

1
scripts/examples-docs.test.mjs
View file

@ -50,7 +50,6 @@ describe('standalone example docs', () => {
config,
/path: \.\.\/\.\.\/packages\/context\/test\/fixtures\/relationship-benchmarks\/orbit_style_product_no_declared_constraints\/data\.sqlite/,
);
assert.match(config, /readonly: true/);
assert.match(config, /llm_proposals: false/);
assert.match(config, /validation_required_for_manifest: true/);
});

1
scripts/installed-live-database-smoke.mjs
View file

@ -92,7 +92,6 @@ export function buildKtxYaml(postgresUrl) {
' warehouse:',
' driver: postgres',
` url: "${postgresUrl}"`,
' readonly: true',
'storage:',
' state: sqlite',
' search: sqlite-fts5',

1
scripts/installed-live-database-smoke.test.mjs
View file

@ -58,7 +58,6 @@ describe('installed live-database artifact smoke helpers', () => {
' warehouse:',
' driver: postgres',
' url: "postgresql://ktx:postgres@127.0.0.1:15432/warehouse"', // pragma: allowlist secret
' readonly: true',
'storage:',
' state: sqlite',
' search: sqlite-fts5',

5
scripts/package-artifacts.mjs
View file

@ -617,7 +617,7 @@ try {
'--skip-sources',
'--skip-agents',
]);
requireProjectStderr('ktx setup', init, projectDir);
requireSuccess('ktx setup', init);
requireOutput('ktx setup', init, /Project: /);
const emptyProjectDir = join(root, 'empty-project');
@ -636,7 +636,7 @@ try {
'--skip-sources',
'--skip-agents',
]);
requireProjectStderr('ktx setup empty project', emptyInit, emptyProjectDir);
requireSuccess('ktx setup empty project', emptyInit);
await writeFile(
join(projectDir, 'ktx.yaml'),
[
@ -645,7 +645,6 @@ try {
' warehouse:',
' driver: sqlite',
' path: warehouse.db',
' readonly: true',
'storage:',
' state: sqlite',
' search: sqlite-fts5',

6
scripts/public-benchmark-manifest.json
View file

@ -4,7 +4,7 @@
"id": "chinook_with_declared_metadata",
"displayName": "Chinook (SQLite, declared metadata)",
"url": "https://raw.githubusercontent.com/lerocha/chinook-database/master/ChinookDatabase/DataSources/Chinook_Sqlite.sqlite",
"sha256": "7651ba378ac2fcd0dfc3c66fb101f7a7eed3ba39a612ec642b96e20702061f15",
"sha256": "7651ba378ac2fcd0dfc3c66fb101f7a7eed3ba39a612ec642b96e20702061f15", "_allowlist": "// pragma: allowlist secret",
"license": "MIT",
"source": "https://github.com/lerocha/chinook-database"
},
@ -12,7 +12,7 @@
"id": "northwind_with_declared_metadata",
"displayName": "Northwind (SQLite, declared metadata)",
"url": "https://github.com/jpwhite3/northwind-SQLite3/raw/main/dist/northwind.db",
"sha256": "2f4f5c68dfcd33ba27373eae48c7a4869800c68095ee0f9f0da494f83382a877",
"sha256": "2f4f5c68dfcd33ba27373eae48c7a4869800c68095ee0f9f0da494f83382a877", "_allowlist": "// pragma: allowlist secret",
"license": "MIT",
"source": "https://github.com/jpwhite3/northwind-SQLite3"
},
@ -20,7 +20,7 @@
"id": "sakila_with_declared_metadata",
"displayName": "Sakila (SQLite, declared metadata)",
"url": "https://raw.githubusercontent.com/bradleygrant/sakila-sqlite3/master/sakila_master.db",
"sha256": "88c91a4a1a6b61f9d3f35904c0a173c887b25e73f20c3c2fdb073818c06f4268",
"sha256": "88c91a4a1a6b61f9d3f35904c0a173c887b25e73f20c3c2fdb073818c06f4268", "_allowlist": "// pragma: allowlist secret",
"license": "BSD-2-Clause",
"source": "https://github.com/bradleygrant/sakila-sqlite3"
},

7
scripts/standalone-ci-workflow.test.mjs
View file

@ -20,6 +20,8 @@ describe('standalone KTX CI workflow', () => {
assertIncludesAll(workflow, [
'permissions:',
'contents: read',
'pre-commit-checks:',
'name: Pre-commit checks',
'typescript-checks:',
'name: TypeScript checks',
'slow-context-tests:',
@ -33,7 +35,7 @@ describe('standalone KTX CI workflow', () => {
'artifact-checks:',
'name: Artifact checks',
'actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd',
'pnpm/action-setup@739bfe42ca9233c5e6aca07c1a25a9d34aca49b0',
'pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093',
'actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e',
'node-version: "24"',
'cache-dependency-path: "pnpm-lock.yaml"',
@ -46,7 +48,10 @@ describe('standalone KTX CI workflow', () => {
'actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405',
'python-version: "3.13"',
'astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b',
'version: "0.11.11"',
'cache-dependency-glob: "uv.lock"',
'uv sync --all-packages --all-groups',
'uv run pre-commit run --all-files',
'uv sync --all-packages',
'uv run pytest',
'pnpm run artifacts:check',

2
uv.lock generated
View file

@ -546,6 +546,7 @@ source = { virtual = "." }
[package.dev-dependencies]
dev = [
{ name = "pre-commit" },
{ name = "pytest" },
{ name = "ruff" },
]
@ -554,6 +555,7 @@ dev = [
[package.metadata.requires-dev]
dev = [
{ name = "pre-commit", specifier = ">=4.6.0" },
{ name = "pytest", specifier = ">=9.0.2" },
{ name = "ruff", specifier = ">=0.8.4" },
]