diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 00000000..430aa729 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,56 @@ +name: Bug report +description: Report something that isn't working +title: "[bug] " +labels: ["bug"] +body: + - type: markdown + attributes: + value: | + For questions or general discussion, use the + [KTX Slack](https://join.slack.com/t/ktxcommunity/shared_invite/zt-3y9b44m1x-LVyNNJD5nwaZHq4XS29LMQ). + - type: textarea + id: bug + attributes: + label: What's the bug? + description: What happened, and what did you expect? + placeholder: When I run `ktx ingest --all`, the Postgres connector fails with X. I expected Y. + validations: + required: true + - type: textarea + id: reproduction + attributes: + label: How can we reproduce it? + description: Commands or steps. A minimal example helps. + render: shell + validations: + required: true + - type: input + id: version + attributes: + label: KTX version + placeholder: "0.x.x" + validations: + required: true + - type: dropdown + id: area + attributes: + label: Which area? + options: + - CLI / setup + - Connector (Postgres) + - Connector (Snowflake) + - Connector (BigQuery) + - Connector (MySQL) + - Connector (SQL Server) + - Connector (SQLite) + - Python semantic layer + - Python daemon + - Docs + - Other + validations: + required: true + - type: textarea + id: extra + attributes: + label: Anything else? + description: OS, Node/Python versions, logs, screenshots. Redact secrets. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000..7a7b5d03 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,14 @@ +blank_issues_enabled: false +contact_links: + - name: KTX Slack — questions and chat + url: https://join.slack.com/t/ktxcommunity/shared_invite/zt-3y9b44m1x-LVyNNJD5nwaZHq4XS29LMQ + about: Ask a question, share what you're building, or get help from maintainers and other users. Faster than filing an issue. + - name: Documentation + url: https://docs.kaelio.com/ktx/docs/ + about: Many setup, configuration, and integration questions are answered here. + - name: Community & Support guide + url: https://docs.kaelio.com/ktx/docs/community/support + about: Full guide on where to ask what — Slack vs. GitHub Issues vs. docs. + - name: Security issues + url: https://github.com/Kaelio/ktx/security/advisories/new + about: Report security vulnerabilities privately via GitHub Security Advisories. Please do not file security issues publicly. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 00000000..5b2f8502 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,39 @@ +name: Feature request +description: Propose a new feature or improvement +title: "[feature] " +labels: ["enhancement"] +body: + - type: markdown + attributes: + value: | + For bug reports, use the bug template. For questions, use the + [KTX Slack](https://join.slack.com/t/ktxcommunity/shared_invite/zt-3y9b44m1x-LVyNNJD5nwaZHq4XS29LMQ). + - type: textarea + id: request + attributes: + label: What do you want, and why? + description: Describe the problem and your proposed solution. + validations: + required: true + - type: dropdown + id: area + attributes: + label: Which area? + options: + - CLI / setup + - Connectors + - Context engine + - Python semantic layer + - Python daemon + - Docs + - Other + validations: + required: true + - type: checkboxes + id: contribute + attributes: + label: Want to contribute this? + options: + - label: Yes, I'd like to open a PR + - label: I'd like guidance, then I'll open a PR + - label: No, just reporting the idea diff --git a/.github/workflows/triage-issues.yml b/.github/workflows/triage-issues.yml new file mode 100644 index 00000000..5a341013 --- /dev/null +++ b/.github/workflows/triage-issues.yml @@ -0,0 +1,28 @@ +name: Triage new issues + +on: + issues: + types: [opened] + +permissions: + issues: write + +jobs: + label-external: + name: Add needs-triage to external issues + runs-on: ubuntu-latest + if: | + github.event.issue.author_association != 'OWNER' && + github.event.issue.author_association != 'MEMBER' && + github.event.issue.author_association != 'COLLABORATOR' + steps: + - name: Apply needs-triage label + uses: actions/github-script@v7 + with: + script: | + await github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + labels: ['needs-triage'], + }); diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..a4fb3040 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,94 @@ +# Contributing to KTX + +Thanks for your interest in KTX. This page covers **how to contribute** and +the **contributor rewards program**. For development setup, repository +layout, and verification commands, see the +[Contributing guide in the docs](https://docs.kaelio.com/ktx/docs/community/contributing). + +## How to contribute + +1. Browse open issues labeled + [`good first issue`](https://github.com/Kaelio/ktx/labels/good%20first%20issue) + or [`help wanted`](https://github.com/Kaelio/ktx/labels/help%20wanted). +2. Comment on the issue to claim it. A maintainer will confirm scope and + assign it to you. +3. For changes not covered by an existing issue, open one first so we can + align on scope before you write code. +4. Open a pull request that resolves the issue. Keep it focused — one + logical change per PR. +5. Run the relevant checks before requesting review. See the + [docs contributing page](https://docs.kaelio.com/ktx/docs/community/contributing#running-tests) + for the right commands per area. + +## Contributor rewards program + +We send merch to contributors whose pull requests get merged. The goal is +to thank the people building KTX with us, not to drive volume. + +### How it works + +1. A maintainer marks an issue `reward:eligible` when it's ready for an + outside contributor. +2. You open a PR that resolves the issue. +3. A maintainer reviews and merges. +4. After merge, the maintainer adds a `reward:tier-*` label and replies + on the PR asking you to email `support@kaelio.com` with your shipping + address, size (if applicable), and a link to the merged PR. +5. We ship within four weeks. + +### Reward tiers + +| Tier | Reward | Earned by | +|------|--------|-----------| +| 1 | Sticker pack | Your first merged PR, any size | +| 2 | T-shirt | A substantive merged PR: bug fix with a regression test, new docs page, connector test fixture, CLI improvement | +| 3 | Hoodie | Three or more merged PRs, or one major contribution (new integration, significant feature) | + +Maintainers decide tier; decisions are final. Tiers do not stack on the +same PR. + +### Eligibility + +- Only **merged** PRs count. Closed-without-merge or stale PRs do not earn + rewards. +- The GitHub account must be at least 30 days old at the time the PR is + opened. +- The PR must resolve a real issue or measurable improvement. +- We ship worldwide where customs allow. If we cannot ship to your region + we will substitute an equivalent (gift card or digital). + +### Not eligible + +- Typo-only PRs and whitespace/formatting changes +- Drive-by style or lint cleanup without prior discussion +- Mass reformatting or wrapper/abstraction churn +- AI-generated PRs that do not pass review on their first revision +- PRs that bundle unrelated changes +- Anything that would be reverted in code review + +We use these rules to keep the program sustainable and to protect the +quality of the project. They are not a judgment on contributors — they +exist so a small maintainer team can keep saying yes. + +## Where to ask what + +See the [Community & Support](https://docs.kaelio.com/ktx/docs/community/support) +page for the full guide. The short version: + +- **Questions, "how do I...", setup help, sharing patterns**: join the + [KTX Slack](https://join.slack.com/t/ktxcommunity/shared_invite/zt-3y9b44m1x-LVyNNJD5nwaZHq4XS29LMQ). +- **Bugs**: use the [Bug report](.github/ISSUE_TEMPLATE/bug_report.yml) + template. +- **Feature requests**: use the + [Feature request](.github/ISSUE_TEMPLATE/feature_request.yml) template. +- **Security**: report privately via + [GitHub Security Advisories](https://github.com/Kaelio/ktx/security/advisories/new), + not as a public issue. + +## Code of conduct + +KTX follows the +[Contributor Covenant](https://www.contributor-covenant.org/version/2/1/code_of_conduct/). +Be respectful, assume good intent, and keep discussion focused on the +project. Report concerns to the maintainers in Slack or by email at +`support@kaelio.com`. diff --git a/README.md b/README.md index d3d55d03..ec6aecb2 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ Codecov Tests Documentation + Join the KTX Slack community License

@@ -94,17 +95,21 @@ Agent integration ready: yes (codex:project) |---------|---------| | `ktx setup` | Create, resume, or update a KTX project | | `ktx status` | Check project readiness | -| `ktx connection list` | List configured connections | +| `ktx connection` | List configured connections | +| `ktx connection test` | Test every configured connection | | `ktx connection test ` | Test one connection | +| `ktx ingest` | Build context for every configured connection | | `ktx ingest ` | Build context for one connection | -| `ktx ingest --all` | Build context for every configured connection | -| `ktx ingest text --connection-id ` | Capture free-form notes into memory | -| `ktx sl list` | List semantic-layer sources | -| `ktx sl search "revenue"` | Search semantic-layer sources | +| `ktx ingest --text "..."` | Capture free-form notes into memory | +| `ktx ingest --file notes.md --connection-id ` | Capture a text file into memory | +| `ktx sl` | List semantic-layer sources | +| `ktx sl "revenue"` | Search semantic-layer sources | | `ktx sl validate --connection-id ` | Validate a semantic source | | `ktx sl query --measure --format sql` | Compile semantic-layer SQL | | `ktx sql --connection "select 1"` | Execute read-only SQL | -| `ktx wiki search "revenue definition"` | Search local wiki context | +| `ktx wiki` | List local wiki pages | +| `ktx wiki "revenue definition"` | Search local wiki context | +| `ktx mcp` | Show MCP daemon status | | `ktx mcp start` | Start the local MCP server for agent clients | Project resolution defaults to `KTX_PROJECT_DIR`, then the nearest `ktx.yaml`, @@ -139,8 +144,8 @@ A typical agent workflow combines wiki and semantic-layer search before querying: ```bash -ktx sl search "revenue" --json -ktx wiki search "refund policy" --json +ktx sl "revenue" --json +ktx wiki "refund policy" --json ktx sl query --connection-id warehouse --measure orders.revenue --format sql ``` @@ -210,8 +215,18 @@ uv run pytest -q - [Quickstart](docs-site/content/docs/getting-started/quickstart.mdx) - [CLI Reference](docs-site/content/docs/cli-reference/ktx.mdx) - [Building Context](docs-site/content/docs/guides/building-context.mdx) +- [Community & Support](docs-site/content/docs/community/support.mdx) - [Contributing](docs-site/content/docs/community/contributing.mdx) +## Community + +- **[Slack](https://join.slack.com/t/ktxcommunity/shared_invite/zt-3y9b44m1x-LVyNNJD5nwaZHq4XS29LMQ)** — ask questions, share what you're building, and chat with maintainers and other users. +- **[GitHub Issues](https://github.com/Kaelio/ktx/issues)** — report bugs and request features. +- **[Contributing guide](docs-site/content/docs/community/contributing.mdx)** — set up the repo, run tests, and open a PR. + +See [Community & Support](docs-site/content/docs/community/support.mdx) for the +full guide on where to ask what. + ## License KTX is licensed under the Apache License, Version 2.0. See `LICENSE`. diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000..da90c1a5 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,31 @@ +# Security Policy + +## Reporting a vulnerability + +If you believe you've found a security vulnerability in KTX, please report it +**privately** through GitHub Security Advisories: + +[Report a vulnerability](https://github.com/Kaelio/ktx/security/advisories/new) + +If you cannot use GitHub Security Advisories, email `support@kaelio.com` +instead. Please do **not** open a public issue, post in the KTX Slack, or +share details elsewhere until we have published a fix. + +When reporting, please include: + +- A description of the issue and its impact +- Steps to reproduce +- The KTX version affected + +## What to expect + +- We will acknowledge your report within a few business days. +- We will work with you to verify the issue and develop a fix. +- We will credit you in the resulting advisory unless you prefer to remain + anonymous. + +## Supported versions + +We provide security fixes for the latest released version of +[`@kaelio/ktx`](https://www.npmjs.com/package/@kaelio/ktx). Older versions +may receive fixes at the maintainers' discretion. diff --git a/docs-site/app/layout.config.tsx b/docs-site/app/layout.config.tsx index ae3bf27b..93b5c45b 100644 --- a/docs-site/app/layout.config.tsx +++ b/docs-site/app/layout.config.tsx @@ -1,10 +1,29 @@ import type { BaseLayoutProps } from "fumadocs-ui/layouts/shared"; +import { GitHubIcon } from "@/components/github-icon"; import { Logo } from "@/components/logo"; +import { SlackIcon } from "@/components/slack-icon"; export const baseOptions: BaseLayoutProps = { nav: { title: , transparentMode: "top", }, - githubUrl: "https://github.com/kaelio/ktx", + links: [ + { + type: "icon", + label: "GitHub", + icon: , + text: "GitHub", + url: "https://github.com/kaelio/ktx", + external: true, + }, + { + type: "icon", + label: "Join the KTX Slack community", + icon: , + text: "Slack", + url: "https://join.slack.com/t/ktxcommunity/shared_invite/zt-3y9b44m1x-LVyNNJD5nwaZHq4XS29LMQ", + external: true, + }, + ], }; diff --git a/docs-site/components/github-icon.tsx b/docs-site/components/github-icon.tsx new file mode 100644 index 00000000..aa3ec564 --- /dev/null +++ b/docs-site/components/github-icon.tsx @@ -0,0 +1,15 @@ +import type { SVGProps } from "react"; + +export function GitHubIcon(props: SVGProps) { + return ( + + ); +} diff --git a/docs-site/components/semantic-layer-flow.tsx b/docs-site/components/semantic-layer-flow.tsx index 770516ad..b6d06dbf 100644 --- a/docs-site/components/semantic-layer-flow.tsx +++ b/docs-site/components/semantic-layer-flow.tsx @@ -4,7 +4,6 @@ import { useCallback, useState } from "react"; import { Background, BackgroundVariant, - Controls, Handle, MarkerType, type Node, @@ -1036,7 +1035,7 @@ export function SemanticLayerFlow() { }} >
- Pan / zoom + Drag to pan • ⌘/Ctrl + scroll to zoom
- ) { + return ( + + ); +} diff --git a/docs-site/content/agents-setup.md b/docs-site/content/agents-setup.md index c7709639..24614ea8 100644 --- a/docs-site/content/agents-setup.md +++ b/docs-site/content/agents-setup.md @@ -5,46 +5,46 @@ Set up KTX from scratch end-to-end as a fully autonomous, agent-driven replaceme # Operating principles - **Be autonomous.** Detect, decide, and act. Only ask the user when you need information that only they can provide: project location, which databases/sources to connect, credentials, and similar choices. -- **Stream short status updates.** Before each major phase ("Checking prerequisites…", "Installing uv…", "Configuring warehouse connection…", "Running fast ingest…") print a one-line update. Not chatty — just enough that the user can see what's happening. +- **Stream short status updates.** Before each major phase ("Checking prerequisites…", "Installing uv…", "Configuring warehouse connection…", "Running fast ingest…") print a one-line update. Not chatty - just enough that the user can see what's happening. - **Verify against docs, never guess.** CLI flags, config keys, and command names must come from the docs or from `ktx --help`. If something looks wrong or missing, say so explicitly. - **Print every command you run and its exit code.** Terse, not silent. -- **Fail loudly with cause + fix.** When a command fails: capture the exact error, identify the cause, change something, retry. Never retry an unchanged command. Exceptions for *known soft-failures* are listed in Phase 4 — handle those without retrying. +- **Fail loudly with cause + fix.** When a command fails: capture the exact error, identify the cause, change something, retry. Never retry an unchanged command. Exceptions for *known soft-failures* are listed in Phase 4 - handle those without retrying. - **No LLM-based ingestion in this flow.** Only `--fast` ingest (schema-only). The user can run `--deep` later. - **Platform-agnostic.** Detect the host OS first and pick the right install commands / path syntax. Anything path- or shell-specific must branch on OS. # Authoritative docs -KTX docs are served at `https://docs.kaelio.com/ktx/`. **Start by fetching `https://docs.kaelio.com/ktx/llms.txt`** to discover the docs map. Scan it for a "troubleshooting" entry — if one exists, read it **before** running install/setup so you can apply known fixes preemptively rather than after failing. If no troubleshooting page is listed (current state of the docs), proceed. Then fetch any other `.md` pages you need (setup, ingest, status, connection types). **Never invent CLI flags or config keys** — verify against the docs or `ktx --help` / `ktx --help`. +KTX docs are served at `https://docs.kaelio.com/ktx/`. **Start by fetching `https://docs.kaelio.com/ktx/llms.txt`** to discover the docs map. Scan it for a "troubleshooting" entry - if one exists, read it **before** running install/setup so you can apply known fixes preemptively rather than after failing. If no troubleshooting page is listed (current state of the docs), proceed. Then fetch any other `.md` pages you need (setup, ingest, status, connection types). **Never invent CLI flags or config keys** - verify against the docs or `ktx --help` / `ktx --help`. -> **Note on the `ktx status` JSON example in the docs.** The docs page for `ktx status` shows an example shaped like `{"title": "...", "checks": [...]}`. That example is outdated. The real CLI output uses a top-level `verdict` field plus a `connections[]` array — see Phase 5 for the canonical success criteria. Trust the shape in this prompt over the docs example. +> **Note on the `ktx status` JSON example in the docs.** The docs page for `ktx status` shows an example shaped like `{"title": "...", "checks": [...]}`. That example is outdated. The real CLI output uses a top-level `verdict` field plus a `connections[]` array - see Phase 5 for the canonical success criteria. Trust the shape in this prompt over the docs example. # Workflow -## Phase 1 — Detect environment +## Phase 1 - Detect environment Determine the host OS (e.g. via `uname -s`, `process.platform`, or `$env:OS`). Use the right install commands per OS for the rest of this flow. | Tool | macOS / Linux | Windows (PowerShell) | |------|---------------|----------------------| | `uv` | `curl -LsSf https://astral.sh/uv/install.sh \| sh` then re-source shell env | `irm https://astral.sh/uv/install.ps1 \| iex` | -| Node.js | use system / fnm / nvm — **do not** auto-install | use system / nvm-windows — **do not** auto-install | +| Node.js | use system / fnm / nvm - **do not** auto-install | use system / nvm-windows - **do not** auto-install | | KTX CLI | `npm install -g …` (see Phase 2) | `npm install -g …` (see Phase 2) | If Node.js is missing, **stop and ask the user** to install it (https://nodejs.org/). Do not attempt to auto-install Node. -## Phase 2 — Verify and install prerequisites +## Phase 2 - Verify and install prerequisites Check each tool in order; install only if missing. -1. **Node.js** — run `node --version`. Require >= 22. If missing or older, stop and instruct the user. -2. **`uv`** — run `uv --version`. If missing, run the OS-appropriate install command, then re-source the shell environment (`export PATH="$HOME/.local/bin:$PATH"` on Linux/macOS) so `uv` is on `PATH`. -3. **KTX CLI** — +1. **Node.js** - run `node --version`. Require >= 22. If missing or older, stop and instruct the user. +2. **`uv`** - run `uv --version`. If missing, run the OS-appropriate install command, then re-source the shell environment (`export PATH="$HOME/.local/bin:$PATH"` on Linux/macOS) so `uv` is on `PATH`. +3. **KTX CLI** - - Install ktx with `npm install -g @kaelio/ktx` - Verify with `ktx --version`. Print one status line per tool ("✓ uv 0.11.15 found", "Installing uv…", "✓ ktx 0.x.y installed"). -## Phase 3 — Gather user choices +## Phase 3 - Gather user choices Ask the user (grouped if your harness supports it; otherwise sequentially): @@ -55,14 +55,14 @@ Ask the user (grouped if your harness supports it; otherwise sequentially): - Connection name (e.g. `warehouse`, `analytics`). - Driver: one of `sqlite`, `postgres`, `mysql`, `sqlserver`, `bigquery`, `snowflake`. - Connection URL/DSN (or service-account file for BigQuery). Accept `env:VAR_NAME` or `file:/abs/path` to avoid pasting raw secrets. - - **Heads-up for the user**: even if they paste a literal URL, KTX will silently relocate it into `/.ktx/secrets/-url` and rewrite `ktx.yaml` to `url: file:…` — this is correct, secure behavior and not a bug. + - **Heads-up for the user**: even if they paste a literal URL, KTX will silently relocate it into `/.ktx/secrets/-url` and rewrite `ktx.yaml` to `url: file:…` - this is correct, secure behavior and not a bug. - Schemas / datasets to include (postgres / sqlserver / snowflake / bigquery only). - Optional `enabled_tables` allowlist if the user wants to scope ingest to specific tables. 5. **BI / metadata sources** (dbt, Metabase, Looker, LookML, MetricFlow, Notion). Default: none. Ask only if the user mentions them. -## Phase 4 — Configure the project +## Phase 4 - Configure the project -Drive the existing wizard non-interactively (verify exact flag names with `ktx setup --help` and the docs — the automation flags are hidden from help but accepted): +Drive the existing wizard non-interactively (verify exact flag names with `ktx setup --help` and the docs - the automation flags are hidden from help but accepted): ``` ktx setup \ @@ -107,17 +107,17 @@ This is **expected** and **does not mean setup failed**. Treat the exit code as - `ktx connection test ` (run next) exits 0 for every connection. - `ktx status --json --no-input` reports `verdict: "ready"`. -If those three conditions hold, proceed to Phase 5 without retrying setup, and **do not** switch to `--deep` to "fix" the readiness gate — deep ingest is explicitly out of scope. Mention this in the final report under "Docs / CLI gaps" so the user is aware. +If those three conditions hold, proceed to Phase 5 without retrying setup, and **do not** switch to `--deep` to "fix" the readiness gate - deep ingest is explicitly out of scope. Mention this in the final report under "Docs / CLI gaps" so the user is aware. -If any of those three conditions do not hold, this is a real failure — capture the error, fetch the relevant docs page, fix the cause, retry. +If any of those three conditions do not hold, this is a real failure - capture the error, fetch the relevant docs page, fix the cause, retry. After `ktx setup` writes `ktx.yaml`, edit it directly for anything flags don't cover: - Per-connection `enabled_tables` allowlist (snake_case, under `connections..enabled_tables`). - Any advanced settings the user requested. -Use a YAML-aware editor (e.g. `uv run python -c "import yaml; …"`) — do not hand-edit blindly. +Use a YAML-aware editor (e.g. `uv run python -c "import yaml; …"`) - do not hand-edit blindly. -## Phase 5 — Verify +## Phase 5 - Verify `ktx setup` already runs a fast schema ingest of every database connection it configures, so you do not need to re-ingest by default. For each configured connection: @@ -139,25 +139,25 @@ Then run the global health check: ktx status --json --no-input ``` -Success requires (canonical shape — supersedes the example in the docs): +Success requires (canonical shape - supersedes the example in the docs): - `verdict: "ready"` at the top of the JSON. - Every `connections[].status === "ok"`. - `ktx connection test ` exited 0 for every connection. -Do **not** run `--deep` ingest in this flow — that requires LLM time and is out of scope. +Do **not** run `--deep` ingest in this flow - that requires LLM time and is out of scope. ### Optional: directly probe the embeddings daemon If the user asks for stronger verification that `sentence-transformers` is actually serving (not just that setup said "ok"), do all of: -1. `ktx dev runtime status --json` → expect `"kind": "ready"` and `"features": [..., "local-embeddings"]`. +1. `ktx admin runtime status --json` → expect `"kind": "ready"` and `"features": [..., "local-embeddings"]`. 2. `pgrep -fa ktx-daemon` → expect a process running `ktx-daemon serve-http`. 3. `curl -sS http://127.0.0.1:/health` → expect HTTP 200 with `{"status":"healthy",…}`. 4. `curl -sS -X POST http://127.0.0.1:/embeddings/compute -H 'content-type: application/json' -d '{"text":"hello"}'` → expect `{"embedding": [...384 floats...]}`. -Discover the port from setup's log line `Started KTX local embeddings daemon: http://127.0.0.1:` or from the daemon's OpenAPI at `GET /openapi.json`. Note: the routes are `/health` and `/embeddings/compute` — not `/healthz` or `/embeddings`. +Discover the port from setup's log line `Started KTX local embeddings daemon: http://127.0.0.1:` or from the daemon's OpenAPI at `GET /openapi.json`. Note: the routes are `/health` and `/embeddings/compute` - not `/healthz` or `/embeddings`. -## Phase 6 — Final report +## Phase 6 - Final report Print a structured report: @@ -180,7 +180,7 @@ Verdict: ready Then **Next steps** (copy-pasteable): 1. Enrich with AI descriptions and embeddings: `ktx ingest --deep` (several minutes per connection). 2. Add more connections later by rerunning this setup or via `ktx setup --database … --database-connection-id …`. -3. Configure BI sources (dbt, Metabase, Looker, LookML, MetricFlow, Notion) — see `ktx setup --help` for `--source …` flags. +3. Configure BI sources (dbt, Metabase, Looker, LookML, MetricFlow, Notion) - see `ktx setup --help` for `--source …` flags. 4. Install agent integration: `ktx setup --agents --target ` (with optional `--global` for `claude-code`/`codex`). 5. Connect the agent / MCP: see docs at `https://docs.kaelio.com/ktx/`. @@ -190,12 +190,12 @@ Under **Docs / CLI gaps to flag** include any of these that applied during your - `ktx status --json` real shape (`verdict`, `connections[]`) doesn't match the example in the docs page. - The pasted DB URL was moved to `.ktx/secrets/-url` automatically. -End with a single line: `RESULT: PASS` or `RESULT: FAIL — `. +End with a single line: `RESULT: PASS` or `RESULT: FAIL - `. # Operating rules (recap) - Print every command you run and its exit code. Status updates may be terse, but never silent. - On failure: capture the error, fetch the relevant docs page, fix the cause, retry. Never retry an unchanged command. -- Known soft-failures (listed in Phase 4 and Phase 5) are not real failures — handle them as documented; do not retry or escalate. +- Known soft-failures (listed in Phase 4 and Phase 5) are not real failures - handle them as documented; do not retry or escalate. - If you find a docs/CLI gap ("docs say X but CLI does Y"), call it out in the final report. -- Never commit credentials — KTX accepts `env:` and `file:` references; prefer those. KTX will also auto-relocate literal URLs into `.ktx/secrets/`, but that does not protect anyone who pasted the URL into chat history. +- Never commit credentials - KTX accepts `env:` and `file:` references; prefer those. KTX will also auto-relocate literal URLs into `.ktx/secrets/`, but that does not protect anyone who pasted the URL into chat history. diff --git a/docs-site/content/docs/cli-reference/ktx-admin.mdx b/docs-site/content/docs/cli-reference/ktx-admin.mdx new file mode 100644 index 00000000..c7df5461 --- /dev/null +++ b/docs-site/content/docs/cli-reference/ktx-admin.mdx @@ -0,0 +1,121 @@ +--- +title: "ktx admin" +description: "Low-level project initialization, runtime, and index management." +--- + +`ktx admin` contains low-level project initialization, managed Python runtime, +and local index management commands. Context building lives at the root as +[`ktx ingest`](/docs/cli-reference/ktx-ingest). Most users should start with +`ktx setup`; use `ktx admin` when preparing local fixtures, checking the bundled +runtime, rebuilding local indexes, or debugging runtime state. + +## Command signature + +```bash +ktx admin [options] +``` + +## Subcommands + +| Subcommand | Description | +|-----------|-------------| +| `init [directory]` | Initialize a Git-backed KTX project directory for maintenance scripts | +| `schema` | Print a JSON Schema describing `ktx.yaml` | +| `runtime` | Install, start, stop, and inspect the KTX-managed Python runtime | +| `reindex` | Sync local wiki and semantic-layer search indexes from disk | + +## `admin init` + +| Flag | Description | Default | +|------|-------------|---------| +| `--force` | Rewrite `ktx.yaml` and scaffold files in an existing project | `false` | + +## `admin schema` + +`ktx admin schema` does not require a `ktx.yaml` file or a configured project +directory. Use it from any directory to generate editor or agent schema files. + +| Flag | Description | Default | +|------|-------------|---------| +| `--output ` | Write the schema to a file instead of stdout | - | + +## `admin runtime` Subcommands + +| Subcommand | Description | +|-----------|-------------| +| `install` | Install the bundled Python runtime wheel into the managed runtime | +| `start` | Start the KTX-managed Python HTTP daemon | +| `stop` | Stop the KTX-managed Python HTTP daemon | +| `status` | Show managed Python runtime status and readiness checks | + +## `admin runtime` Options + +| Flag | Description | Default | +|------|-------------|---------| +| `--feature ` | Runtime feature level for `install` and `start` (`core` or `local-embeddings`) | `core` | +| `--json` | Print JSON output for `status` | `false` | +| `--yes` | Accepted by `install` for scripted install commands | `false` | +| `--force` | Reinstall for `install`, or restart for `start` | `false` | +| `--all` | Stop all recorded or discoverable KTX daemon processes with `stop` | `false` | + +## Examples + +```bash +ktx admin init +ktx admin init ./my-project +ktx admin init --force + +ktx admin schema +ktx admin schema --output ./ktx.schema.json + +ktx admin runtime install --yes +ktx admin runtime install --feature local-embeddings --yes +ktx admin runtime status +ktx admin runtime start +ktx admin runtime start --feature local-embeddings +ktx admin runtime stop +ktx admin runtime stop --all + +ktx admin reindex +ktx admin reindex --force +ktx admin reindex --output plain +ktx admin reindex --json +``` + +## Output + +Runtime commands print the runtime root, installed features, daemon URL, daemon +pid, and log paths where relevant. `ktx admin runtime status --json` includes the +runtime status plus readiness checks. + +## `admin reindex` + +`ktx admin reindex` syncs local wiki and semantic-layer search indexes from +files on disk into `.ktx/db.sqlite`. The command discovers `wiki/global/`, each +`wiki/user//` directory, and each `semantic-layer//` +directory except `_schema`. + +```bash +ktx admin reindex +ktx admin reindex --force +ktx admin reindex --output plain +ktx admin reindex --json +``` + +By default, KTX compares stored search text with the files on disk. It only +re-embeds changed rows and removes rows for files that no longer exist. With +`--force`, KTX clears each discovered scope first and then rebuilds it. + +When embeddings are not configured, KTX still writes lexical FTS rows and +prints an embeddings warning. If a scope fails, KTX keeps processing the +remaining scopes and exits with code `1` after output is written. If the local +state database cannot open or the configured managed embedding runtime is +missing, KTX prints the error and exits with code `1`. + +## Common errors + +| Error | Cause | Recovery | +|-------|-------|----------| +| Runtime status reports missing pieces | Packages, Python environment, or linked CLI are not ready | Run `pnpm install`, `pnpm run setup:dev`, `uv sync --all-groups`, then `ktx admin runtime status` | +| Runtime daemon does not start | The managed Python runtime is missing or stale | Run `ktx admin runtime install --yes`, then `ktx admin runtime start` | +| Multiple daemon processes remain | Older daemon state files or stray processes exist | Run `ktx admin runtime stop --all`, then start the runtime again | diff --git a/docs-site/content/docs/cli-reference/ktx-connection.mdx b/docs-site/content/docs/cli-reference/ktx-connection.mdx index 2d61451f..5c4c5324 100644 --- a/docs-site/content/docs/cli-reference/ktx-connection.mdx +++ b/docs-site/content/docs/cli-reference/ktx-connection.mdx @@ -10,15 +10,21 @@ systems. Use `ktx setup` to add, remove, or reconfigure them. ## Command signature ```bash -ktx connection [options] +ktx connection # list all configured connections +ktx connection list # explicit list +ktx connection test [connectionId] # test one (or all, when omitted) ``` +Bare `ktx connection` lists configured connections. `ktx connection test` +with no positional and no flag tests every configured connection. + ## Subcommands | Subcommand | Description | |-----------|-------------| +| (none) | List configured connections (alias for `list`) | | `list` | List configured connections | -| `test [connectionId]` | Test one configured connection, or every connection with `--all` | +| `test [connectionId]` | Test one configured connection; omit the id (or pass `--all`) to test every connection | ## Options @@ -29,7 +35,7 @@ ktx connection [options] | Flag | Description | Default | |------|-------------|---------| -| `--all` | Test every configured connection and print a summary list | `false` | +| `--all` | Test every configured connection and print a summary list | implicit when no `connectionId` is supplied | Project directory resolution defaults to `KTX_PROJECT_DIR`, then the nearest `ktx.yaml`, then the current working directory. @@ -38,12 +44,15 @@ Project directory resolution defaults to `KTX_PROJECT_DIR`, then the nearest ```bash # List all configured connections -ktx connection list - -# Test a connection -ktx connection test my-warehouse +ktx connection # Test every configured connection +ktx connection test + +# Test one connection +ktx connection test my-warehouse + +# Test every connection explicitly ktx connection test --all # Test a connection from outside the project @@ -58,7 +67,8 @@ Metabase mapping prompts for BI-to-warehouse mappings. ## Output -`ktx connection list` prints a table of configured ids and drivers. +`ktx connection` (or `ktx connection list`) prints a table of configured ids +and drivers. ```text ID DRIVER @@ -76,8 +86,8 @@ Driver: postgres Status: ok ``` -`ktx connection test --all` prints one row per configured connection and exits -non-zero if any probe fails. +`ktx connection test` (bare) and `ktx connection test --all` print one row per +configured connection and exit non-zero if any probe fails. ```text ╭ connection test --all diff --git a/docs-site/content/docs/cli-reference/ktx-dev.mdx b/docs-site/content/docs/cli-reference/ktx-dev.mdx deleted file mode 100644 index efa3d74b..00000000 --- a/docs-site/content/docs/cli-reference/ktx-dev.mdx +++ /dev/null @@ -1,91 +0,0 @@ ---- -title: "ktx dev" -description: "Low-level project initialization and runtime management." ---- - -`ktx dev` contains low-level project initialization and managed Python runtime -commands. Context building lives at the root as -[`ktx ingest`](/docs/cli-reference/ktx-ingest). Most users should start with -`ktx setup`; use `ktx dev` when preparing local fixtures, checking the bundled -runtime, or debugging runtime state. - -## Command signature - -```bash -ktx dev [options] -``` - -## Subcommands - -| Subcommand | Description | -|-----------|-------------| -| `init [directory]` | Initialize a Git-backed KTX project directory for maintenance scripts | -| `schema` | Print a JSON Schema describing `ktx.yaml` | -| `runtime` | Install, start, stop, and inspect the KTX-managed Python runtime | - -## `dev init` - -| Flag | Description | Default | -|------|-------------|---------| -| `--force` | Rewrite `ktx.yaml` and scaffold files in an existing project | `false` | - -## `dev schema` - -`ktx dev schema` does not require a `ktx.yaml` file or a configured project -directory. Use it from any directory to generate editor or agent schema files. - -| Flag | Description | Default | -|------|-------------|---------| -| `--output ` | Write the schema to a file instead of stdout | - | - -## `dev runtime` Subcommands - -| Subcommand | Description | -|-----------|-------------| -| `install` | Install the bundled Python runtime wheel into the managed runtime | -| `start` | Start the KTX-managed Python HTTP daemon | -| `stop` | Stop the KTX-managed Python HTTP daemon | -| `status` | Show managed Python runtime status and readiness checks | - -## `dev runtime` Options - -| Flag | Description | Default | -|------|-------------|---------| -| `--feature ` | Runtime feature level for `install` and `start` (`core` or `local-embeddings`) | `core` | -| `--json` | Print JSON output for `status` | `false` | -| `--yes` | Accepted by `install` for scripted install commands | `false` | -| `--force` | Reinstall for `install`, or restart for `start` | `false` | -| `--all` | Stop all recorded or discoverable KTX daemon processes with `stop` | `false` | - -## Examples - -```bash -ktx dev init -ktx dev init ./my-project -ktx dev init --force - -ktx dev schema -ktx dev schema --output ./ktx.schema.json - -ktx dev runtime install --yes -ktx dev runtime install --feature local-embeddings --yes -ktx dev runtime status -ktx dev runtime start -ktx dev runtime start --feature local-embeddings -ktx dev runtime stop -ktx dev runtime stop --all -``` - -## Output - -Runtime commands print the runtime root, installed features, daemon URL, daemon -pid, and log paths where relevant. `ktx dev runtime status --json` includes the -runtime status plus readiness checks. - -## Common errors - -| Error | Cause | Recovery | -|-------|-------|----------| -| Runtime status reports missing pieces | Packages, Python environment, or linked CLI are not ready | Run `pnpm install`, `pnpm run setup:dev`, `uv sync --all-groups`, then `ktx dev runtime status` | -| Runtime daemon does not start | The managed Python runtime is missing or stale | Run `ktx dev runtime install --yes`, then `ktx dev runtime start` | -| Multiple daemon processes remain | Older daemon state files or stray processes exist | Run `ktx dev runtime stop --all`, then start the runtime again | diff --git a/docs-site/content/docs/cli-reference/ktx-ingest.mdx b/docs-site/content/docs/cli-reference/ktx-ingest.mdx index 49485d10..49b176b5 100644 --- a/docs-site/content/docs/cli-reference/ktx-ingest.mdx +++ b/docs-site/content/docs/cli-reference/ktx-ingest.mdx @@ -1,35 +1,44 @@ --- title: "ktx ingest" -description: "Build or refresh KTX context from configured connections." +description: "Build or refresh KTX context, or capture text into KTX memory." --- -`ktx ingest` builds or refreshes KTX context from configured connections. -Database connections build schema context. Context-source connections ingest -metadata from tools such as dbt, Looker, Metabase, MetricFlow, LookML, and -Notion. The current public command is connection-centric: pass one -`connectionId`, or pass `--all`. +`ktx ingest` builds or refreshes KTX context from configured connections, and +can also capture free-form text into KTX memory. Database connections build +schema context. Context-source connections ingest metadata from tools such as +dbt, Looker, Metabase, MetricFlow, LookML, and Notion. Pass `--text` or +`--file` to capture inline text or text files into memory instead. ## Command signature ```bash ktx ingest [options] [connectionId] -ktx ingest text [options] [files...] ``` -Use a connection id to build one configured connection. Use `--all` to build -every configured connection. Database connections run before context-source -connections when you use `--all`. +- Bare `ktx ingest` (no positional, no `--all`) ingests every configured + connection. +- `ktx ingest ` ingests one configured connection. +- `ktx ingest --text "..."` (or `--file `) captures notes into KTX + memory instead of ingesting a connection. -## `ktx ingest` Options +Database connections run before context-source connections when more than one +connection is selected. + +## Options | Flag | Description | Default | |------|-------------|---------| -| `--all` | Ingest all configured connections | `false` | +| `--all` | Ingest all configured connections (same as bare invocation) | `false` | | `--fast` | Use deterministic database schema ingest | Stored connection default, or `fast` | | `--deep` | Use AI-enriched database ingest | Stored connection default, or `fast` | | `--query-history` | Include database query-history usage patterns | Stored connection default | | `--no-query-history` | Skip database query-history usage patterns for this run | Stored connection default | | `--query-history-window-days ` | BigQuery/Snowflake query-history lookback window for this run | Stored connection default | +| `--text ` | Capture inline text into KTX memory; repeatable | `[]` | +| `--file ` | Capture a text file into KTX memory; use `-` for stdin; repeatable | `[]` | +| `--connection-id ` | KTX connection id to tag captured text/file notes | - | +| `--user-id ` | Memory user id for text/file capture attribution | `local-cli` | +| `--fail-fast` | Stop after the first failed text/file item | `false` | | `--plain` | Print plain text output | `true` | | `--json` | Print JSON output | `false` | | `--yes` | Install required managed runtime features without prompting | `false` | @@ -42,8 +51,8 @@ Postgres reads the current `pg_stat_statements` aggregate data instead of a time-windowed history table. Query-history ingest runs after schema ingest and requires deep ingest readiness. -When `--all` selects both databases and context sources, database ingest runs -first, then source ingest and memory updates run for source connections. +When more than one connection is selected, database ingest runs first, then +source ingest and memory updates run for source connections. Some ingest paths use the managed KTX Python runtime. Query-history ingest uses it for SQL analysis, and Looker source ingest uses it for Looker identifier @@ -51,23 +60,15 @@ parsing. In an interactive terminal, `ktx ingest` prompts before installing the required runtime features. Use `--yes` to install them without prompting, or use `--no-input` to fail fast with install guidance. -## `ktx ingest text` Options - -Use `ktx ingest text` to capture free-form text artifacts into KTX memory. -Provide files, pass `--text` one or more times, or use `-` as a file argument to -read one item from stdin. - -| Flag | Description | Default | -|------|-------------|---------| -| `--text ` | Text content to ingest; repeat for a batch | `[]` | -| `--connection-id ` | Optional KTX connection id for semantic-layer capture | - | -| `--user-id ` | Memory user id for capture attribution | `local-cli` | -| `--json` | Print JSON output | `false` | -| `--fail-fast` | Stop after the first failed text item | `false` | +`--text` and `--file` cannot be combined with a positional `connectionId` or +`--all`; pass `--connection-id ` instead to tag captured notes. ## Examples ```bash +# Build every configured connection (bare = --all) +ktx ingest + # Build one database or source connection ktx ingest warehouse @@ -85,15 +86,17 @@ ktx ingest warehouse --query-history-window-days 30 # Build a source connection ktx ingest notion -# Build all configured connections -ktx ingest --all -ktx ingest --all --deep +# Capture inline text into memory +ktx ingest --text "Refunds are excluded from net revenue." -# Capture local Markdown notes into memory -ktx ingest text docs/revenue-notes.md --connection-id warehouse +# Capture multiple text snippets in one call +ktx ingest --text "Revenue is gross receipts." --text "Orders are completed purchases." + +# Capture a local Markdown file into memory and tag it to a connection +ktx ingest --file docs/revenue-notes.md --connection-id warehouse # Capture one stdin item -printf "Refunds are excluded from net revenue." | ktx ingest text - +printf "Refunds are excluded from net revenue." | ktx ingest --file - ``` ## Output @@ -153,7 +156,6 @@ KTX_INGEST_TRACE_LEVEL=trace ktx ingest metabase | Connection not configured | The connection id is not present in `ktx.yaml` | Add the connection with `ktx setup` or update `ktx.yaml` | | Deep readiness is missing | `--deep` or query history needs model, embedding, and scan-enrichment configuration | Run `ktx setup` or rerun with `--fast` | | Query history is unsupported | The selected database driver does not support query history | Run schema ingest without query-history flags | -| Python runtime is missing | The selected ingest target needs runtime-backed SQL analysis or source parsing | Accept the interactive prompt, rerun with `--yes`, or run the suggested `ktx dev runtime install` command | -| No ingest target was selected | No connection id was provided and `--all` was omitted | Run `ktx ingest ` or `ktx ingest --all` | +| Python runtime is missing | The selected ingest target needs runtime-backed SQL analysis or source parsing | Accept the interactive prompt, rerun with `--yes`, or run the suggested `ktx admin runtime install` command | | Source options were ignored | Depth and query-history flags were supplied for a non-database source | Omit database-only flags when ingesting source connections | | Text ingest stops early | `--fail-fast` was used and one item failed | Fix the failed item or rerun without `--fail-fast` to collect all failures | diff --git a/docs-site/content/docs/cli-reference/ktx-setup.mdx b/docs-site/content/docs/cli-reference/ktx-setup.mdx index 6dbe54c2..e98a51e4 100644 --- a/docs-site/content/docs/cli-reference/ktx-setup.mdx +++ b/docs-site/content/docs/cli-reference/ktx-setup.mdx @@ -231,7 +231,7 @@ Use `ktx status` for repeatable readiness checks after setup exits. | Setup resumes an unexpected project | `KTX_PROJECT_DIR` or nearest `ktx.yaml` points to another directory | Pass `--project-dir ` explicitly | | Setup cannot run in CI | Required values are missing and `--no-input` disables prompts | Provide the relevant automation flags or create a fixture `ktx.yaml` | | Provider health check fails | Provider key, model id, Vertex project, or Vertex location is invalid | Fix the `env:` or `file:` reference and rerun setup | -| Python runtime is missing | The selected setup needs runtime-backed agent, query-history, Looker, or local embedding features | Accept the interactive prompt, rerun with `--yes`, or run the suggested `ktx dev runtime install` command | +| Python runtime is missing | The selected setup needs runtime-backed agent, query-history, Looker, or local embedding features | Accept the interactive prompt, rerun with `--yes`, or run the suggested `ktx admin runtime install` command | | `--enable-query-history` is rejected | The selected database driver does not support query history | Use Postgres, BigQuery, or Snowflake, or rerun without query-history flags | | Source setup rejects location flags | Both `--source-path` and `--source-git-url` were supplied | Choose the local path or the Git URL, not both | | Agent integration missing | Setup skipped the agents step | Run `ktx setup --agents --target ` | diff --git a/docs-site/content/docs/cli-reference/ktx-sl.mdx b/docs-site/content/docs/cli-reference/ktx-sl.mdx index f395a170..e0c6a101 100644 --- a/docs-site/content/docs/cli-reference/ktx-sl.mdx +++ b/docs-site/content/docs/cli-reference/ktx-sl.mdx @@ -10,34 +10,33 @@ the vocabulary agents use to generate correct SQL. ## Command signature ```bash -ktx sl [options] +ktx sl [options] [query...] # list (bare) or search (with query) +ktx sl validate [options] +ktx sl query [options] ``` +- Bare `ktx sl` lists semantic-layer sources. +- `ktx sl ` searches semantic-layer sources (multi-word queries are + joined with a space). +- `ktx sl validate` and `ktx sl query` remain as explicit subcommands. + ## Subcommands | Subcommand | Description | |-----------|-------------| -| `list` | List semantic-layer sources | -| `search ` | Search semantic-layer sources | +| (none, no query) | List semantic-layer sources | +| (none, with query) | Search semantic-layer sources | | `validate ` | Validate a semantic-layer source against the database schema | | `query` | Compile or execute a Semantic Query | ## Options -### `sl list` +### `sl` (list or search) | Flag | Description | Default | |------|-------------|---------| | `--connection-id ` | Filter by KTX connection id | - | -| `--output ` | Output mode: `pretty` (default in TTY), `plain` (TSV), or `json` | `pretty` | -| `--json` | Shortcut for `--output=json` (overrides `--output`) | `false` | - -### `sl search` - -| Flag | Description | Default | -|------|-------------|---------| -| `--connection-id ` | Filter by KTX connection id | - | -| `--limit ` | Maximum search results | - | +| `--limit ` | Maximum search results (search mode only) | - | | `--output ` | Output mode: `pretty` (default in TTY), `plain` (TSV), or `json` | `pretty` | | `--json` | Shortcut for `--output=json` (overrides `--output`) | `false` | @@ -73,16 +72,16 @@ ktx sl [options] ```bash # List all semantic sources -ktx sl list +ktx sl # List sources for a specific connection -ktx sl list --connection-id my-warehouse +ktx sl --connection-id my-warehouse # List sources as JSON -ktx sl list --json +ktx sl --json # Search sources as JSON -ktx sl search "revenue" --json +ktx sl "revenue" --json # Validate a source against the live schema ktx sl validate orders --connection-id my-warehouse @@ -137,13 +136,13 @@ ktx sl query \ ## Output -Semantic-layer list and search commands return human-readable output by -default. Use `--json` on `list` or `search` when an agent needs structured -output. Use `--format sql` on `query` to inspect generated SQL before -execution, or leave `--format json` for the compiled query and optional rows. -Pretty `sl search` output shows `#1`, `#2`, and later rank badges for the -displayed results. Plain and JSON output keep the raw `score` value, which is a -ranking score rather than a percentage. +Bare `ktx sl` (list) and `ktx sl ` (search) return human-readable +output by default. Use `--json` when an agent needs structured output. Use +`--format sql` on `query` to inspect generated SQL before execution, or leave +`--format json` for the compiled query and optional rows. Pretty search output +shows `#1`, `#2`, and later rank badges for the displayed results. Plain and +JSON output keep the raw `score` value, which is a ranking score rather than a +percentage. ```json { @@ -161,8 +160,8 @@ ranking score rather than a percentage. | Error | Cause | Recovery | |-------|-------|----------| -| Source not found | Source name or connection id is wrong | Run `ktx sl list --json` and retry with an exact source name and connection id | +| Source not found | Source name or connection id is wrong | Run `ktx sl --json` and retry with an exact source name and connection id | | Validation fails | YAML references missing columns, invalid joins, or invalid SQL expressions | Fix the source YAML and rerun `ktx sl validate` | -| Query compile fails | Measure, dimension, filter, or segment name is invalid | Search sources with `ktx sl search`, inspect the source YAML in your project files, then retry using declared fields | +| Query compile fails | Measure, dimension, filter, or segment name is invalid | Search sources with `ktx sl `, inspect the source YAML in your project files, then retry using declared fields | | Execution returns too many rows | `--max-rows` is missing or too high | Add `--max-rows` with a bounded value before executing | -| Runtime install is blocked | Query execution needs the managed Python runtime and prompts are disabled | Run `ktx dev runtime install --feature core --yes`, or rerun `ktx sl query --yes` | +| Runtime install is blocked | Query execution needs the managed Python runtime and prompts are disabled | Run `ktx admin runtime install --feature core --yes`, or rerun `ktx sl query --yes` | diff --git a/docs-site/content/docs/cli-reference/ktx-wiki.mdx b/docs-site/content/docs/cli-reference/ktx-wiki.mdx index e1caabb5..aac60a07 100644 --- a/docs-site/content/docs/cli-reference/ktx-wiki.mdx +++ b/docs-site/content/docs/cli-reference/ktx-wiki.mdx @@ -10,42 +10,28 @@ them for context when answering questions about your data. ## Command signature ```bash -ktx wiki [options] +ktx wiki [options] [query...] ``` -## Subcommands +- Bare `ktx wiki` lists local wiki pages. +- `ktx wiki ` searches local wiki pages (multi-word queries are + joined with a space). -| Subcommand | Description | -|-----------|-------------| -| `list` | List local wiki pages | -| `search ` | Search local wiki pages | - -The current public CLI lists and searches wiki pages. Edit the Markdown files -under `wiki/` directly, or ingest source content with `ktx ingest`, when you -need to add or update wiki knowledge. +Edit the Markdown files under `wiki/` directly, or ingest source content with +`ktx ingest`, when you need to add or update wiki knowledge. ## Options -### `wiki list` - | Flag | Description | Default | |------|-------------|---------| | `--user-id ` | Local user id | `local` | +| `--limit ` | Maximum search results (search mode only) | - | | `--output ` | Output mode: `pretty` (default in TTY), `plain` (TSV), or `json` | `pretty` | | `--json` | Shortcut for `--output=json` (overrides `--output`) | `false` | -### `wiki search` - -| Flag | Description | Default | -|------|-------------|---------| -| `--user-id ` | Local user id | `local` | -| `--limit ` | Maximum search results | - | -| `--output ` | Output mode: `pretty` (default in TTY), `plain` (TSV), or `json` | `pretty` | -| `--json` | Shortcut for `--output=json` (overrides `--output`) | `false` | - -`wiki search` uses hybrid search when `storage.search` is `sqlite-fts5`. KTX -combines lexical SQLite FTS5 matches, token matches, and semantic matches from -wiki page embeddings stored in `.ktx/db.sqlite`. If embeddings are not +`ktx wiki ` uses hybrid search when `storage.search` is `sqlite-fts5`. +KTX combines lexical SQLite FTS5 matches, token matches, and semantic matches +from wiki page embeddings stored in `.ktx/db.sqlite`. If embeddings are not configured or the embedding backend is unavailable, KTX skips the semantic lane and keeps lexical and token results. @@ -53,22 +39,22 @@ and keeps lexical and token results. ```bash # List all wiki pages -ktx wiki list +ktx wiki # List all wiki pages as JSON -ktx wiki list --json +ktx wiki --json # Search wiki pages -ktx wiki search "monthly recurring revenue" +ktx wiki "monthly recurring revenue" # Search wiki pages as JSON -ktx wiki search "monthly recurring revenue" --json --limit 10 +ktx wiki "monthly recurring revenue" --json --limit 10 # Print search results as TSV -ktx wiki search "monthly recurring revenue" --output plain +ktx wiki "monthly recurring revenue" --output plain # Inspect which search lanes were used -ktx --debug wiki search "monthly recurring revenue" --json +ktx --debug wiki "monthly recurring revenue" --json ``` ## Output diff --git a/docs-site/content/docs/cli-reference/ktx.mdx b/docs-site/content/docs/cli-reference/ktx.mdx index 937d2529..65cdc5ff 100644 --- a/docs-site/content/docs/cli-reference/ktx.mdx +++ b/docs-site/content/docs/cli-reference/ktx.mdx @@ -48,7 +48,7 @@ ktx stop status logs - dev + admin init [directory] schema runtime @@ -56,6 +56,7 @@ ktx start stop status + reindex ``` The public context-build entrypoint is `ktx ingest [connectionId]` or @@ -90,11 +91,11 @@ ktx status ktx ingest warehouse # Build every configured connection -ktx ingest --all +ktx ingest # Search semantic-layer sources and wiki pages -ktx sl search "revenue" -ktx wiki search "revenue recognition" +ktx sl "revenue" +ktx wiki "revenue recognition" # Execute read-only SQL ktx sql --connection warehouse "select count(*) from public.orders" diff --git a/docs-site/content/docs/cli-reference/meta.json b/docs-site/content/docs/cli-reference/meta.json index 6385c7d4..49eb8ba7 100644 --- a/docs-site/content/docs/cli-reference/meta.json +++ b/docs-site/content/docs/cli-reference/meta.json @@ -11,6 +11,6 @@ "ktx-wiki", "ktx-status", "ktx-mcp", - "ktx-dev" + "ktx-admin" ] } diff --git a/docs-site/content/docs/community/meta.json b/docs-site/content/docs/community/meta.json index 0f97320e..e181be6c 100644 --- a/docs-site/content/docs/community/meta.json +++ b/docs-site/content/docs/community/meta.json @@ -1,5 +1,5 @@ { "title": "Community", "defaultOpen": true, - "pages": ["contributing"] + "pages": ["support", "contributing"] } diff --git a/docs-site/content/docs/community/support.mdx b/docs-site/content/docs/community/support.mdx new file mode 100644 index 00000000..d53e168c --- /dev/null +++ b/docs-site/content/docs/community/support.mdx @@ -0,0 +1,47 @@ +--- +title: Community & Support +description: Join the KTX Slack community, report bugs, and get help. +--- + +KTX is an open-source project. The community is where users, contributors, and +the core team trade questions, share patterns, and shape the roadmap. + +## Where to go + +| You want to... | Go here | +|----------------|---------| +| Ask a question or chat with the community | [KTX Slack](https://join.slack.com/t/ktxcommunity/shared_invite/zt-3y9b44m1x-LVyNNJD5nwaZHq4XS29LMQ) | +| Report a bug or request a feature | [GitHub Issues](https://github.com/Kaelio/ktx/issues) | +| Read or contribute to the docs | [docs.kaelio.com/ktx](https://docs.kaelio.com/ktx/docs/) | +| Contribute code | [Contributing guide](/docs/community/contributing) | + +## Slack + +Join the KTX Slack to ask questions, share what you're building, and get help +from maintainers and other users. + +[**Join the KTX Slack →**](https://join.slack.com/t/ktxcommunity/shared_invite/zt-3y9b44m1x-LVyNNJD5nwaZHq4XS29LMQ) + +Slack is the right place for: + +- **Setup and configuration questions** that don't fit a bug report +- **Quick "how do I..."** questions +- **Sharing patterns** for prompts, semantic-layer definitions, or agent workflows +- **Feedback** on the roadmap and early features + +For anything reproducible - a crash, a wrong result, an unexpected CLI error - +open a [GitHub issue](https://github.com/Kaelio/ktx/issues) instead. Issues are +searchable, get triaged, and stay attached to the eventual fix. + +## GitHub + +- **[Issues](https://github.com/Kaelio/ktx/issues)** - bugs and feature requests +- **[Pull requests](https://github.com/Kaelio/ktx/pulls)** - code, docs, and connector contributions +- **[Releases](https://github.com/Kaelio/ktx/releases)** - changelog and published versions + +## Code of conduct + +KTX follows the [Contributor Covenant](https://www.contributor-covenant.org/version/2/1/code_of_conduct/). +Be respectful, assume good intent, and keep discussion focused on the project. +Report conduct concerns to the maintainers in Slack or by email at +`support@kaelio.com`. diff --git a/docs-site/content/docs/concepts/semantic-layer-internals.mdx b/docs-site/content/docs/concepts/semantic-layer-internals.mdx index a011d1cb..f64836fe 100644 --- a/docs-site/content/docs/concepts/semantic-layer-internals.mdx +++ b/docs-site/content/docs/concepts/semantic-layer-internals.mdx @@ -6,7 +6,7 @@ description: How KTX compiles a short Semantic Query into safe, dialect-correct import { SemanticLayerFlow } from "@/components/semantic-layer-flow"; KTX's semantic layer is a compiler that turns intent into SQL. The agent -declares _what_ it wants — measures, dimensions, filters — in a small +declares _what_ it wants - measures, dimensions, filters - in a small Semantic Query. KTX figures out the _how_: which tables to join, what grain to aggregate at, how to keep fan-out from inflating measures, and what dialect the warehouse speaks. @@ -21,8 +21,8 @@ This page covers four mechanics: ## Imperative SQL vs declarative Semantic Querying Writing analytics SQL is imperative work. Every question forces the -agent to hold two things in mind at once: _what_ it wants — a measure, a -slice, a filter — and _how_ to compute it: which tables to join, which +agent to hold two things in mind at once: _what_ it wants - a measure, a +slice, a filter - and _how_ to compute it: which tables to join, which key links them, what grain to aggregate at, how to keep one fact from inflating another, and what dialect the warehouse speaks. Plumbing on top of intent, every query. @@ -30,7 +30,7 @@ top of intent, every query. KTX's semantic layer separates those concerns: - **You and KTX maintain the how.** Sources, joins, grain, measures, and - segments live in reviewable YAML — the analytical contract the team + segments live in reviewable YAML - the analytical contract the team agrees on, version-controlled. - **The agent declares the what.** It sends a Semantic Query and trusts the compiler to produce safe SQL. diff --git a/docs-site/content/docs/getting-started/introduction.mdx b/docs-site/content/docs/getting-started/introduction.mdx index d0ee126d..2bf6e9b1 100644 --- a/docs-site/content/docs/getting-started/introduction.mdx +++ b/docs-site/content/docs/getting-started/introduction.mdx @@ -95,3 +95,12 @@ best first step for users; contributor setup lives in the community docs. Machine-readable docs and agent-facing setup notes. + +## Community + +Have questions, want to share what you're building, or chat with maintainers? +Join the [KTX Slack community](https://join.slack.com/t/ktxcommunity/shared_invite/zt-3y9b44m1x-LVyNNJD5nwaZHq4XS29LMQ). +For bug reports and feature requests, open a +[GitHub issue](https://github.com/Kaelio/ktx/issues). See +[Community & Support](/docs/community/support) for the full guide on where to +ask what. diff --git a/docs-site/content/docs/getting-started/quickstart.mdx b/docs-site/content/docs/getting-started/quickstart.mdx index d0566ec9..14fe476f 100644 --- a/docs-site/content/docs/getting-started/quickstart.mdx +++ b/docs-site/content/docs/getting-started/quickstart.mdx @@ -126,8 +126,8 @@ If you choose local `sentence-transformers` embeddings, KTX uses the managed Python runtime. To prepare it before setup, run: ```bash -ktx dev runtime install --feature local-embeddings --yes -ktx dev runtime start --feature local-embeddings +ktx admin runtime install --feature local-embeddings --yes +ktx admin runtime start --feature local-embeddings ``` During the database step, setup tests the saved connection and builds initial diff --git a/docs-site/content/docs/guides/building-context.mdx b/docs-site/content/docs/guides/building-context.mdx index 39edaa85..b1d5ed36 100644 --- a/docs-site/content/docs/guides/building-context.mdx +++ b/docs-site/content/docs/guides/building-context.mdx @@ -106,28 +106,30 @@ edits. ## Text ingest -Use `ktx ingest text` for notes, Markdown, runbooks, Slack exports, or other -searchable memory. +Use `ktx ingest --text` / `ktx ingest --file` for notes, Markdown, runbooks, +Slack exports, or other searchable memory. ```bash # Capture a Markdown file -ktx ingest text docs/revenue-notes.md --connection-id warehouse +ktx ingest --file docs/revenue-notes.md --connection-id warehouse # Capture one stdin item -printf "Refunds are excluded from net revenue." | ktx ingest text - +printf "Refunds are excluded from net revenue." | ktx ingest --file - # Capture direct text -ktx ingest text --text "ARR excludes one-time implementation fees." +ktx ingest --text "ARR excludes one-time implementation fees." ``` Useful flags: | Flag | Description | |------|-------------| +| `--text ` | Capture inline text into memory; repeatable | +| `--file ` | Capture a text file (or `-` for stdin) into memory; repeatable | | `--connection-id ` | Attach the captured memory to a KTX connection | | `--user-id ` | Attribute capture to a user scope, default `local-cli` | | `--json` | Print structured output | -| `--fail-fast` | Stop after the first failed text item | +| `--fail-fast` | Stop after the first failed text/file item | Use text ingest for small, high-signal documents. Prefer configured source ingest for Notion, dbt, Metabase, and similar systems. @@ -165,8 +167,8 @@ Then inspect what changed: ```bash git status --short -ktx sl list --json -ktx wiki search "revenue" --json --limit 10 +ktx sl --json +ktx wiki "revenue" --json --limit 10 ``` ## Common errors @@ -176,6 +178,6 @@ ktx wiki search "revenue" --json --limit 10 | Connection not configured | The connection id is missing from `ktx.yaml` | Add it with `ktx setup` | | Deep readiness is missing | LLM or embeddings are not setup-ready | Run `ktx setup`, or rerun with `--fast` | | Query history is unsupported | The selected database driver does not expose query history | Run schema ingest without query-history flags | -| No target selected | You omitted both a connection id and `--all` | Run `ktx ingest ` or `ktx ingest --all` | +| No connections configured | The project has no entries under `connections` | Run `ktx setup` and add a database or source connection | | Source flags have no effect | Depth and query-history flags were supplied for a source connector | Use those flags only for database connections | | Text ingest stops early | `--fail-fast` stopped on the first failed item | Fix the item or rerun without `--fail-fast` | diff --git a/docs-site/content/docs/guides/serving-agents.mdx b/docs-site/content/docs/guides/serving-agents.mdx index 8710d3ba..df6eba2a 100644 --- a/docs-site/content/docs/guides/serving-agents.mdx +++ b/docs-site/content/docs/guides/serving-agents.mdx @@ -58,9 +58,9 @@ context-build, and agent-integration readiness. ### Semantic layer discovery ```bash -ktx sl list --json -ktx sl list --connection-id warehouse --json -ktx sl search "revenue" --json --limit 10 +ktx sl --json +ktx sl --connection-id warehouse --json +ktx sl "revenue" --json --limit 10 ``` Use these commands to find source names, connection ids, measures, dimensions, @@ -99,8 +99,8 @@ For complex calls, agents can write a JSON query object and pass it with ### Wiki context ```bash -ktx wiki list --json -ktx wiki search "revenue recognition" --json --limit 10 +ktx wiki --json +ktx wiki "revenue recognition" --json --limit 10 ``` Search wiki context for business definitions, metric caveats, process rules, and @@ -112,8 +112,8 @@ Agents can refresh context when the user asks them to: ```bash ktx ingest warehouse --fast -ktx ingest --all -ktx ingest text docs/revenue-notes.md --connection-id warehouse +ktx ingest +ktx ingest --file docs/revenue-notes.md --connection-id warehouse ``` Use `--deep` only when LLM and embedding setup is ready. @@ -123,7 +123,7 @@ Use `--deep` only when LLM and embedding setup is ready. Agents should: - Run `ktx status --json` before using KTX context. -- Use `ktx sl search` and `ktx wiki search` before writing SQL from memory. +- Use `ktx sl ` and `ktx wiki ` before writing SQL from memory. - Inspect the relevant YAML or Markdown files after search returns candidates. - Compile SQL with `ktx sl query --format sql` before executing. - Use `--max-rows` whenever executing a live query. @@ -156,5 +156,5 @@ For per-client notes, see [Agent Clients](/docs/integrations/agent-clients). | Agent says KTX is unavailable | Agent did not load the generated instruction file | Rerun `ktx setup --agents --target ` and restart the agent session | | Agent command cannot find the project | Agent is running outside the KTX directory | Add `--project-dir ` or open the agent in the project root | | Generated rules point at a missing CLI path | CLI was moved, rebuilt, or reinstalled | Rerun `ktx setup --agents` | -| Agent cannot find a metric | Context is missing or stale | Run `ktx sl search`, inspect source YAML, then refresh with `ktx ingest` if needed | +| Agent cannot find a metric | Context is missing or stale | Run `ktx sl `, inspect source YAML, then refresh with `ktx ingest` if needed | | Agent query returns too many rows | The command executed without a result cap | Require `--max-rows` for executed queries | diff --git a/docs-site/content/docs/guides/writing-context.mdx b/docs-site/content/docs/guides/writing-context.mdx index 2b9824c8..066c4dea 100644 --- a/docs-site/content/docs/guides/writing-context.mdx +++ b/docs-site/content/docs/guides/writing-context.mdx @@ -13,9 +13,9 @@ Use this order for most context changes: 1. Discover existing context. ```bash - ktx sl list --json - ktx sl search "revenue" --json - ktx wiki search "revenue recognition" --json --limit 10 + ktx sl --json + ktx sl "revenue" --json + ktx wiki "revenue recognition" --json --limit 10 ``` 2. Edit the smallest relevant files under `semantic-layer//` or @@ -306,7 +306,7 @@ Useful frontmatter: 1. Search first. ```bash - ktx wiki search "active customer definition" --json --limit 10 + ktx wiki "active customer definition" --json --limit 10 ``` 2. If no page covers the rule, create or edit a Markdown file under @@ -323,8 +323,8 @@ Before accepting agent-written context: ```bash git diff -- semantic-layer wiki ktx sl validate orders --connection-id warehouse -ktx sl search "revenue" --json -ktx wiki search "revenue recognition" --json --limit 10 +ktx sl "revenue" --json +ktx wiki "revenue recognition" --json --limit 10 ``` Check definitions, hidden columns, join relationships, and generated SQL. diff --git a/docs-site/content/docs/integrations/agent-clients.mdx b/docs-site/content/docs/integrations/agent-clients.mdx index 4a670315..9f1ad659 100644 --- a/docs-site/content/docs/integrations/agent-clients.mdx +++ b/docs-site/content/docs/integrations/agent-clients.mdx @@ -130,10 +130,10 @@ description: Use local KTX semantic context and wiki knowledge for this project. Available commands: - `ktx status --json --project-dir /path/to/project` -- `ktx sl list --json --project-dir /path/to/project` -- `ktx sl search '' --json --project-dir /path/to/project --connection-id ''` +- `ktx sl --json --project-dir /path/to/project` +- `ktx sl '' --json --project-dir /path/to/project --connection-id ''` - `ktx sl query --project-dir /path/to/project --connection-id '' --query-file '' --format json --execute --max-rows 100` -- `ktx wiki search '' --json --project-dir /path/to/project --limit 10` +- `ktx wiki '' --json --project-dir /path/to/project --limit 10` ``` ### Workflow tips @@ -281,9 +281,9 @@ Admin CLI skills call the same KTX CLI commands: | Command | Description | |---------|-------------| | `ktx status --json` | Return project setup and context readiness | -| `ktx wiki search --json` | Search wiki pages | -| `ktx sl list --json` | List semantic-layer sources | -| `ktx sl search --json` | Search semantic-layer sources | +| `ktx wiki --json` | Search wiki pages | +| `ktx sl --json` | List semantic-layer sources | +| `ktx sl --json` | Search semantic-layer sources | | `ktx sl validate --connection-id ` | Validate semantic source definitions | | `ktx sl query --format json` | Execute a Semantic Query when semantic compute is configured | diff --git a/examples/package-artifacts/README.md b/examples/package-artifacts/README.md index 22ecaf92..ed72f13f 100644 --- a/examples/package-artifacts/README.md +++ b/examples/package-artifacts/README.md @@ -12,8 +12,8 @@ imports the package entry point, and runs installed `ktx` commands against a generated local project. The managed Python runtime smoke requires `uv` on `PATH`, isolates -`KTX_RUNTIME_ROOT`, verifies `ktx dev runtime status`, runs `ktx sl query --yes` to -install the core runtime from the bundled wheel, checks `ktx dev runtime status`, +`KTX_RUNTIME_ROOT`, verifies `ktx admin runtime status`, runs `ktx sl query --yes` to +install the core runtime from the bundled wheel, checks `ktx admin runtime status`, starts and reuses the managed daemon, and stops it. The artifact manifest contains the public `@kaelio/ktx` npm tarball and the diff --git a/packages/cli/src/admin-reindex.test.ts b/packages/cli/src/admin-reindex.test.ts new file mode 100644 index 00000000..eb75c651 --- /dev/null +++ b/packages/cli/src/admin-reindex.test.ts @@ -0,0 +1,145 @@ +import type { ReindexSummary } from '@ktx/context/index-sync'; +import { describe, expect, it, vi } from 'vitest'; +import { renderReindexJson, renderReindexPlain, reindexHasErrors } from './admin-reindex.js'; +import { runKtxCli } from './index.js'; + +function makeIo(options: { stdoutIsTTY?: boolean } = {}) { + let stdout = ''; + let stderr = ''; + return { + io: { + stdout: { + isTTY: options.stdoutIsTTY, + write: (chunk: string) => { + stdout += chunk; + }, + }, + stderr: { + write: (chunk: string) => { + stderr += chunk; + }, + }, + }, + stdout: () => stdout, + stderr: () => stderr, + }; +} + +function summary(overrides: Partial = {}): ReindexSummary { + return { + scopes: [ + { + kind: 'wiki', + label: 'global', + scope: 'global', + scopeId: null, + scanned: 42, + updated: 3, + deleted: 1, + embeddingsRecomputed: 3, + embeddingsFailed: 0, + durationMs: 412, + }, + { + kind: 'sl', + label: 'warehouse', + connectionId: 'warehouse', + scanned: 18, + updated: 2, + deleted: 0, + embeddingsRecomputed: 2, + embeddingsFailed: 0, + durationMs: 287, + }, + ], + totals: { scanned: 60, updated: 5, deleted: 1, embeddingsRecomputed: 5, embeddingsFailed: 0 }, + dbPath: '.ktx/db.sqlite', + force: false, + embeddingsAvailable: true, + durationMs: 1234, + ...overrides, + }; +} + +describe('admin reindex renderers', () => { + it('renders plain scope lines to stderr and summary to stdout', () => { + const io = makeIo(); + + renderReindexPlain(summary(), io.io); + + expect(io.stderr()).toContain('wiki/global\tscanned=42\tupdated=3\tdeleted=1\tembeddings=3\tduration_ms=412\n'); + expect(io.stderr()).toContain('sl/warehouse\tscanned=18\tupdated=2\tdeleted=0\tembeddings=2\tduration_ms=287\n'); + expect(io.stdout()).toBe('reindex\tscopes=2\tscanned=60\tupdated=5\tdeleted=1\tembeddings=5\tduration_ms=1234\n'); + }); + + it('renders rebuilt labels in plain force mode', () => { + const io = makeIo(); + + renderReindexPlain(summary({ force: true }), io.io); + + expect(io.stderr()).toContain('rebuilt=3'); + expect(io.stdout()).toContain('rebuilt=5'); + expect(io.stdout()).not.toContain('updated=5'); + }); + + it('renders json envelope to stdout only', () => { + const io = makeIo(); + + renderReindexJson(summary(), io.io); + + expect(JSON.parse(io.stdout())).toMatchObject({ + kind: 'reindex', + data: { totals: { scanned: 60, updated: 5 } }, + meta: { command: 'admin reindex' }, + }); + expect(io.stderr()).toBe(''); + }); + + it('detects per-scope errors', () => { + expect( + reindexHasErrors( + summary({ + scopes: [{ ...summary().scopes[0]!, error: 'provider failed' }], + }), + ), + ).toBe(true); + }); +}); + +describe('admin reindex Commander routing', () => { + it('routes flags to the injectable reindex runner', async () => { + const { mkdir, mkdtemp, rm, writeFile } = await import('node:fs/promises'); + const { tmpdir } = await import('node:os'); + const { join } = await import('node:path'); + const tempDir = await mkdtemp(join(tmpdir(), 'ktx-admin-reindex-cli-')); + const projectDir = join(tempDir, 'project'); + const io = makeIo(); + const adminReindex = vi.fn(async () => 0); + + try { + await mkdir(projectDir, { recursive: true }); + await writeFile(join(projectDir, 'ktx.yaml'), '{}\n', 'utf-8'); + + await expect( + runKtxCli( + ['--project-dir', projectDir, 'admin', 'reindex', '--force', '--json', '--output', 'plain'], + io.io, + { adminReindex }, + ), + ).resolves.toBe(0); + } finally { + await rm(tempDir, { recursive: true, force: true }); + } + + expect(adminReindex).toHaveBeenCalledWith( + { + projectDir, + force: true, + json: true, + output: 'plain', + cliVersion: '0.1.0-rc.1', + }, + io.io, + ); + }); +}); diff --git a/packages/cli/src/admin-reindex.ts b/packages/cli/src/admin-reindex.ts new file mode 100644 index 00000000..8518fc2c --- /dev/null +++ b/packages/cli/src/admin-reindex.ts @@ -0,0 +1,210 @@ +import { + createLocalKtxEmbeddingProviderFromConfig, + KtxIngestEmbeddingPortAdapter, + MANAGED_SENTENCE_TRANSFORMERS_BASE_URL, + type KtxEmbeddingPort, +} from '@ktx/context'; +import { reindexLocalIndexes, type ReindexScopeResult, type ReindexSummary } from '@ktx/context/index-sync'; +import { loadKtxProject, type KtxLocalProject } from '@ktx/context/project'; +import { Option, type Command } from '@commander-js/extra-typings'; +import { cancel, intro, log, note, outro } from '@clack/prompts'; +import type { KtxCliCommandContext } from './cli-program.js'; +import type { KtxCliIo } from './cli-runtime.js'; +import { resolveOutputMode } from './io/mode.js'; +import { green, red, SYMBOLS } from './io/symbols.js'; +import { ensureManagedLocalEmbeddingsDaemon } from './managed-local-embeddings.js'; + +export interface KtxAdminReindexArgs { + projectDir: string; + force: boolean; + output?: 'pretty' | 'plain' | 'json'; + json?: boolean; + cliVersion: string; +} + +export function registerAdminReindexCommand(admin: Command, context: KtxCliCommandContext): void { + admin + .command('reindex') + .description('Sync local wiki and semantic-layer search indexes from disk') + .option('--force', 'Clear each discovered scope before rebuilding it', false) + .option('--json', 'Shortcut for --output=json (overrides --output)', false) + .addOption( + new Option('--output ', 'Output mode: pretty, plain, or json').choices(['pretty', 'plain', 'json']), + ) + .action(async (options: { force?: boolean; json?: boolean; output?: 'pretty' | 'plain' | 'json' }, command) => { + const runner = context.deps.adminReindex ?? runKtxAdminReindex; + const { resolveCommandProjectDir } = await import('./cli-program.js'); + context.setExitCode( + await runner( + { + projectDir: resolveCommandProjectDir(command), + force: options.force === true, + json: options.json === true, + output: options.output, + cliVersion: context.packageInfo.version, + }, + context.io, + ), + ); + }); +} + +async function resolveReindexEmbeddingService( + project: KtxLocalProject, + args: KtxAdminReindexArgs, + io: KtxCliIo, +): Promise { + const config = project.config.ingest.embeddings; + if (config.backend === 'none') { + return null; + } + + if ( + config.backend === 'sentence-transformers' && + config.sentenceTransformers?.base_url === MANAGED_SENTENCE_TRANSFORMERS_BASE_URL + ) { + const daemon = await ensureManagedLocalEmbeddingsDaemon({ + cliVersion: args.cliVersion, + projectDir: project.projectDir, + installPolicy: 'never', + io, + }); + const provider = createLocalKtxEmbeddingProviderFromConfig(config, { env: { ...process.env, ...daemon.env } }); + return provider ? new KtxIngestEmbeddingPortAdapter(provider) : null; + } + + const provider = createLocalKtxEmbeddingProviderFromConfig(config); + return provider ? new KtxIngestEmbeddingPortAdapter(provider) : null; +} + +function scopeKey(scope: ReindexScopeResult): string { + if (scope.kind === 'wiki') { + return scope.scope === 'user' ? `wiki/user/${scope.scopeId ?? 'local'}` : 'wiki/global'; + } + return `sl/${scope.connectionId ?? scope.label}`; +} + +function quotePlainValue(value: string): string { + return `"${value.replaceAll('\\', '\\\\').replaceAll('"', '\\"')}"`; +} + +export function reindexHasErrors(summary: ReindexSummary): boolean { + return summary.scopes.some((scope) => scope.error); +} + +export function renderReindexPlain(summary: ReindexSummary, io: KtxCliIo): void { + const updateKey = summary.force ? 'rebuilt' : 'updated'; + for (const scope of summary.scopes) { + const cells = [ + scopeKey(scope), + `scanned=${scope.scanned}`, + `${updateKey}=${scope.updated}`, + `deleted=${scope.deleted}`, + `embeddings=${summary.embeddingsAvailable ? String(scope.embeddingsRecomputed) : '-'}`, + `duration_ms=${scope.durationMs}`, + ...(scope.error ? [`error=${quotePlainValue(scope.error)}`] : []), + ]; + io.stderr.write(`${cells.join('\t')}\n`); + } + const failed = summary.scopes.filter((scope) => scope.error).length; + io.stdout.write( + [ + 'reindex', + `scopes=${summary.scopes.length}`, + `scanned=${summary.totals.scanned}`, + `${updateKey}=${summary.totals.updated}`, + `deleted=${summary.totals.deleted}`, + `embeddings=${summary.embeddingsAvailable ? String(summary.totals.embeddingsRecomputed) : '-'}`, + `duration_ms=${summary.durationMs}`, + ...(failed > 0 ? [`failed=${failed}`] : []), + ].join('\t') + '\n', + ); +} + +export function renderReindexJson(summary: ReindexSummary, io: KtxCliIo): void { + io.stdout.write(`${JSON.stringify({ kind: 'reindex', data: summary, meta: { command: 'admin reindex' } }, null, 2)}\n`); +} + +function noun(scope: ReindexScopeResult): string { + return scope.kind === 'wiki' ? 'pages' : 'sources'; +} + +function formatScopeLine(scope: ReindexScopeResult, force: boolean, embeddingsAvailable: boolean): string { + if (scope.error) { + return `${scope.kind === 'wiki' ? 'Wiki' : 'SL'}: ${scope.label} ${SYMBOLS.emDash} failed: ${scope.error}`; + } + const changedLabel = force ? 'rebuilt' : 'updated'; + const parts = [`${scope.scanned} ${noun(scope)}`]; + if (scope.updated > 0) { + parts.push(`${scope.updated} ${changedLabel}`); + } else { + parts.push('unchanged'); + } + if (!force && scope.deleted > 0) { + parts.push(`${scope.deleted} deleted`); + } + if (embeddingsAvailable) { + parts.push(`${scope.embeddingsRecomputed} embeddings recomputed`); + } + parts.push(`${scope.durationMs}ms`); + return `${scope.kind === 'wiki' ? 'Wiki' : 'SL'}: ${scope.label} ${SYMBOLS.emDash} ${parts.join(` ${SYMBOLS.middot} `)}`; +} + +function renderReindexPretty(summary: ReindexSummary, io: KtxCliIo): void { + intro(summary.force ? 'ktx admin reindex --force' : 'ktx admin reindex'); + if (!summary.embeddingsAvailable) { + log.warn(`Embeddings: not configured ${SYMBOLS.emDash} indexing lexical only`); + } + for (const scope of summary.scopes) { + const line = formatScopeLine(scope, summary.force, summary.embeddingsAvailable); + if (scope.error) { + log.error(red(line)); + } else { + log.success(green(line)); + } + } + const failed = summary.scopes.filter((scope) => scope.error).length; + note( + [ + `scopes ${summary.scopes.length}`, + `scanned ${summary.totals.scanned}`, + `${summary.force ? 'rebuilt' : 'updated'} ${summary.totals.updated}`, + `deleted ${summary.totals.deleted}`, + `embeddings ${summary.embeddingsAvailable ? summary.totals.embeddingsRecomputed : SYMBOLS.emDash}`, + `index ${summary.dbPath}`, + ...(failed > 0 ? [`failed ${failed}`] : []), + ].join('\n'), + 'Summary', + ); + if (failed > 0) { + cancel(`reindex completed with ${failed} error${failed === 1 ? '' : 's'}`); + } else { + outro(`Done in ${(summary.durationMs / 1000).toFixed(1)}s`); + } + void io; +} + +async function runKtxAdminReindex(args: KtxAdminReindexArgs, io: KtxCliIo = process): Promise { + try { + const project = await loadKtxProject({ projectDir: args.projectDir }); + const embeddingService = await resolveReindexEmbeddingService(project, args, io); + const summary = await reindexLocalIndexes(project, { force: args.force, embeddingService }); + const mode = resolveOutputMode({ explicit: args.output, json: args.json, io }); + + if (!summary.embeddingsAvailable && mode === 'plain') { + io.stderr.write(`Embeddings: not configured ${SYMBOLS.emDash} indexing lexical only\n`); + } + + if (mode === 'json') { + renderReindexJson(summary, io); + } else if (mode === 'plain') { + renderReindexPlain(summary, io); + } else { + renderReindexPretty(summary, io); + } + return reindexHasErrors(summary) ? 1 : 0; + } catch (error) { + io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + return 1; + } +} diff --git a/packages/cli/src/dev.test.ts b/packages/cli/src/admin.test.ts similarity index 75% rename from packages/cli/src/dev.test.ts rename to packages/cli/src/admin.test.ts index 95adb20c..15f4179e 100644 --- a/packages/cli/src/dev.test.ts +++ b/packages/cli/src/admin.test.ts @@ -22,14 +22,14 @@ function makeIo() { }; } -describe('dev Commander tree', () => { - it('prints visible dev help with only supported low-level command groups', async () => { +describe('admin Commander tree', () => { + it('prints visible admin help with supported low-level command groups', async () => { const testIo = makeIo(); - await expect(runKtxCli(['dev', '--help'], testIo.io)).resolves.toBe(0); + await expect(runKtxCli(['admin', '--help'], testIo.io)).resolves.toBe(0); - expect(testIo.stdout()).toContain('Usage: ktx dev [options] [command]'); - for (const command of ['init', 'runtime']) { + expect(testIo.stdout()).toContain('Usage: ktx admin [options] [command]'); + for (const command of ['init', 'runtime', 'reindex']) { expect(testIo.stdout()).toContain(command); } for (const removed of [ @@ -52,27 +52,35 @@ describe('dev Commander tree', () => { expect(testIo.stderr()).toBe(''); }); - it('lists dev in root command rows', async () => { + it('lists admin in root command rows', async () => { const testIo = makeIo(); await expect(runKtxCli(['--help'], testIo.io)).resolves.toBe(0); expect(testIo.stdout()).not.toContain('Advanced:'); - expect(testIo.stdout()).toContain('dev'); - expect(testIo.stdout()).toMatch(/Low-level project initialization and runtime\s+management/); + expect(testIo.stdout()).toContain('admin'); + expect(testIo.stdout()).toMatch(/Low-level project initialization,\s+runtime,\s+and index management/); expect(testIo.stderr()).toBe(''); }); - it('keeps project scaffolding under dev init', async () => { + it('does not keep a dev alias', async () => { + const testIo = makeIo(); + + await expect(runKtxCli(['dev', '--help'], testIo.io)).resolves.toBe(1); + + expect(testIo.stderr()).toContain("unknown command 'dev'"); + }); + + it('keeps project scaffolding under admin init', async () => { const { mkdtemp, readFile, rm } = await import('node:fs/promises'); const { tmpdir } = await import('node:os'); const { join } = await import('node:path'); - const tempDir = await mkdtemp(join(tmpdir(), 'ktx-dev-init-')); + const tempDir = await mkdtemp(join(tmpdir(), 'ktx-admin-init-')); const projectDir = join(tempDir, 'warehouse'); const testIo = makeIo(); try { - await expect(runKtxCli(['dev', 'init', projectDir], testIo.io)).resolves.toBe(0); + await expect(runKtxCli(['admin', 'init', projectDir], testIo.io)).resolves.toBe(0); expect(testIo.stdout()).toContain(`Initialized KTX project at ${projectDir}`); await expect(readFile(join(projectDir, 'ktx.yaml'), 'utf-8')).resolves.not.toContain('project:'); @@ -82,17 +90,17 @@ describe('dev Commander tree', () => { } }); - it('uses global project-dir for dev init when the positional directory is omitted', async () => { + it('uses global project-dir for admin init when the positional directory is omitted', async () => { const { mkdtemp, rm } = await import('node:fs/promises'); const { tmpdir } = await import('node:os'); const { join } = await import('node:path'); - const tempDir = await mkdtemp(join(tmpdir(), 'ktx-dev-init-global-')); + const tempDir = await mkdtemp(join(tmpdir(), 'ktx-admin-init-global-')); const projectDir = join(tempDir, 'global-init'); const testIo = makeIo(); try { await expect( - runKtxCli(['--project-dir', projectDir, 'dev', 'init'], testIo.io), + runKtxCli(['--project-dir', projectDir, 'admin', 'init'], testIo.io), ).resolves.toBe(0); expect(testIo.stdout()).toContain(`Initialized KTX project at ${projectDir}`); @@ -106,7 +114,7 @@ describe('dev Commander tree', () => { const { mkdtemp, rm } = await import('node:fs/promises'); const { tmpdir } = await import('node:os'); const { join } = await import('node:path'); - const tempDir = await mkdtemp(join(tmpdir(), 'ktx-dev-schema-')); + const tempDir = await mkdtemp(join(tmpdir(), 'ktx-admin-schema-')); const missingProjectDir = join(tempDir, 'missing-project'); const originalProjectDir = process.env.KTX_PROJECT_DIR; const testIo = makeIo(); @@ -114,7 +122,7 @@ describe('dev Commander tree', () => { try { process.env.KTX_PROJECT_DIR = missingProjectDir; - await expect(runKtxCli(['dev', 'schema'], testIo.io)).resolves.toBe(0); + await expect(runKtxCli(['admin', 'schema'], testIo.io)).resolves.toBe(0); expect(JSON.parse(testIo.stdout())).toMatchObject({ title: 'ktx.yaml', @@ -131,19 +139,19 @@ describe('dev Commander tree', () => { } }); - it('rejects removed dev command groups', async () => { + it('rejects removed admin command groups', async () => { for (const argv of [ - ['dev', 'doctor', 'setup'], - ['dev', 'runtime', 'doctor'], - ['dev', 'runtime', 'prune', '--dry-run'], - ['dev', 'scan', 'warehouse'], - ['dev', 'ingest', 'run'], - ['dev', 'mapping', 'list'], - ['dev', 'completion', 'zsh'], - ['dev', '__complete', '--shell', 'zsh', '--position', '2', '--', 'ktx', ''], - ['dev', 'knowledge', 'list'], - ['dev', 'model', 'list'], - ['dev', 'artifacts'], + ['admin', 'doctor', 'setup'], + ['admin', 'runtime', 'doctor'], + ['admin', 'runtime', 'prune', '--dry-run'], + ['admin', 'scan', 'warehouse'], + ['admin', 'ingest', 'run'], + ['admin', 'mapping', 'list'], + ['admin', 'completion', 'zsh'], + ['admin', '__complete', '--shell', 'zsh', '--position', '2', '--', 'ktx', ''], + ['admin', 'knowledge', 'list'], + ['admin', 'model', 'list'], + ['admin', 'artifacts'], ]) { const testIo = makeIo(); @@ -155,8 +163,8 @@ describe('dev Commander tree', () => { it.each([ { - argv: ['dev', 'runtime', '--help'], - expected: ['Usage: ktx dev runtime', 'install', 'start', 'stop', 'status'], + argv: ['admin', 'runtime', '--help'], + expected: ['Usage: ktx admin runtime', 'install', 'start', 'stop', 'status'], }, ])('prints generated nested help for $argv', async ({ argv, expected }) => { const io = makeIo(); @@ -167,7 +175,7 @@ describe('dev Commander tree', () => { for (const text of expected) { expect(io.stdout()).toContain(text); } - if (argv.join(' ') === 'dev runtime --help') { + if (argv.join(' ') === 'admin runtime --help') { expect(io.stdout()).not.toContain('prune'); expect(io.stdout()).not.toContain('doctor'); } diff --git a/packages/cli/src/dev.ts b/packages/cli/src/admin.ts similarity index 76% rename from packages/cli/src/dev.ts rename to packages/cli/src/admin.ts index 12ad6f46..af23c192 100644 --- a/packages/cli/src/dev.ts +++ b/packages/cli/src/admin.ts @@ -1,27 +1,28 @@ import { resolve } from 'node:path'; import type { Command } from '@commander-js/extra-typings'; import { type CommandWithGlobalOptions, type KtxCliCommandContext, resolveCommandProjectDir } from './cli-program.js'; +import { registerAdminReindexCommand } from './admin-reindex.js'; import { registerRuntimeCommands } from './commands/runtime-commands.js'; import { profileMark } from './startup-profile.js'; -profileMark('module:dev'); +profileMark('module:admin'); -export function registerDevCommands(program: Command, context: KtxCliCommandContext): void { - const dev = program - .command('dev') - .description('Low-level project initialization and runtime management') +export function registerAdminCommands(program: Command, context: KtxCliCommandContext): void { + const admin = program + .command('admin') + .description('Low-level project initialization, runtime, and index management') .showHelpAfterError(); - dev.hook('preAction', (_thisCommand, actionCommand) => { - context.writeDebug?.('dev', actionCommand); + admin.hook('preAction', (_thisCommand, actionCommand) => { + context.writeDebug?.('admin', actionCommand); }); - dev.action(() => { - dev.outputHelp(); + admin.action(() => { + admin.outputHelp(); context.setExitCode(0); }); - dev + admin .command('init') .description('Initialize a Git-backed KTX project directory for maintenance scripts') .argument('[directory]', 'Project directory') @@ -44,7 +45,7 @@ export function registerDevCommands(program: Command, context: KtxCliCommandCont }, ); - dev + admin .command('schema') .description('Print a JSON Schema describing ktx.yaml (for editors and LLM agents)') .option('--output ', 'Write the schema to a file instead of stdout') @@ -62,5 +63,6 @@ export function registerDevCommands(program: Command, context: KtxCliCommandCont context.setExitCode(0); }); - registerRuntimeCommands(dev, context); + registerRuntimeCommands(admin, context); + registerAdminReindexCommand(admin, context); } diff --git a/packages/cli/src/cli-program.test.ts b/packages/cli/src/cli-program.test.ts index f0ac9595..2e5333c9 100644 --- a/packages/cli/src/cli-program.test.ts +++ b/packages/cli/src/cli-program.test.ts @@ -31,7 +31,7 @@ describe('buildKtxProgram', () => { expect(program.name()).toBe('ktx'); const topLevel = program.commands.map((command) => command.name()).sort(); - for (const expected of ['setup', 'connection', 'ingest', 'sl', 'dev']) { + for (const expected of ['setup', 'connection', 'ingest', 'sl', 'admin']) { expect(topLevel).toContain(expected); } }); diff --git a/packages/cli/src/cli-program.ts b/packages/cli/src/cli-program.ts index 34b13854..84f740f5 100644 --- a/packages/cli/src/cli-program.ts +++ b/packages/cli/src/cli-program.ts @@ -10,7 +10,7 @@ import { registerSetupCommands } from './commands/setup-commands.js'; import { registerSlCommands } from './commands/sl-commands.js'; import { registerSqlCommands } from './commands/sql-commands.js'; import { registerStatusCommands } from './commands/status-commands.js'; -import { registerDevCommands } from './dev.js'; +import { registerAdminCommands } from './admin.js'; import { renderMissingProjectMessage } from './doctor.js'; import { findNearestKtxProjectDir, resolveKtxProjectDir } from './project-resolver.js'; import { profileMark, profileSpan } from './startup-profile.js'; @@ -58,8 +58,8 @@ type CommandPathNode = CommandWithGlobalOptions & { }; const PROJECT_AWARE_ROOT_COMMANDS = new Set(['setup', 'connection', 'ingest', 'wiki', 'sl', 'sql', 'status', 'mcp']); -const PROJECT_INDEPENDENT_DEV_COMMANDS = new Set(['runtime', 'schema']); -const COMMANDS_THAT_CREATE_PROJECT = new Set(['setup', 'ktx dev init']); +const PROJECT_INDEPENDENT_ADMIN_COMMANDS = new Set(['runtime', 'schema']); +const COMMANDS_THAT_CREATE_PROJECT = new Set(['setup', 'ktx admin init']); const COMMANDS_WITH_OWN_MISSING_PROJECT_HANDLING = new Set(['status']); const GLOBAL_OPTIONS_WITH_VALUE = new Set(['--project-dir']); const GLOBAL_OPTIONS_WITHOUT_VALUE = new Set(['--debug', '--help', '-h', '--version', '-v']); @@ -172,15 +172,15 @@ function isProjectAwareCommand(path: string[]): boolean { } const rootCommand = path[1]; - if (rootCommand === 'dev') { - return path[2] !== undefined && !PROJECT_INDEPENDENT_DEV_COMMANDS.has(path[2]); + if (rootCommand === 'admin') { + return path[2] !== undefined && !PROJECT_INDEPENDENT_ADMIN_COMMANDS.has(path[2]); } return rootCommand !== undefined && PROJECT_AWARE_ROOT_COMMANDS.has(rootCommand); } function shouldSuppressProjectDirLine(path: string[], options: Record): boolean { const commandPathKey = path.join(' '); - if (commandPathKey === 'ktx dev init') { + if (commandPathKey === 'ktx admin init') { return true; } @@ -421,7 +421,7 @@ export function buildKtxProgram(options: BuildKtxProgramOptions): Command { registerSqlCommands(program, context); registerStatusCommands(program, context); registerMcpCommands(program, context); - registerDevCommands(program, context); + registerAdminCommands(program, context); return program; } diff --git a/packages/cli/src/cli-runtime.ts b/packages/cli/src/cli-runtime.ts index b8bc636d..a2d4765c 100644 --- a/packages/cli/src/cli-runtime.ts +++ b/packages/cli/src/cli-runtime.ts @@ -1,6 +1,7 @@ import { createRequire } from 'node:module'; import type { KtxConnectionArgs } from './connection.js'; +import type { KtxAdminReindexArgs } from './admin-reindex.js'; import type { KtxDoctorArgs } from './doctor.js'; import type { KtxKnowledgeArgs } from './knowledge.js'; import type { KtxPublicIngestArgs } from './public-ingest.js'; @@ -30,6 +31,7 @@ export interface KtxCliIo { } export interface KtxCliDeps { + adminReindex?: (args: KtxAdminReindexArgs, io: KtxCliIo) => Promise; setup?: (args: KtxSetupArgs, io: KtxCliIo) => Promise; connection?: (args: KtxConnectionArgs, io: KtxCliIo) => Promise; doctor?: (args: KtxDoctorArgs, io: KtxCliIo) => Promise; diff --git a/packages/cli/src/commands/connection-commands.ts b/packages/cli/src/commands/connection-commands.ts index 181e8905..213bf608 100644 --- a/packages/cli/src/commands/connection-commands.ts +++ b/packages/cli/src/commands/connection-commands.ts @@ -2,6 +2,7 @@ import { type Command } from '@commander-js/extra-typings'; import { type KtxCliCommandContext, resolveCommandProjectDir } from '../cli-program.js'; import type { KtxConnectionArgs } from '../connection.js'; import { profileMark } from '../startup-profile.js'; +import { resolveConnectionSelection } from './connection-selection.js'; profileMark('module:commands/connection-commands'); @@ -18,7 +19,10 @@ export function registerConnectionCommands(program: Command, context: KtxCliComm .addHelpText( 'after', '\nProject directory defaults to KTX_PROJECT_DIR when set, otherwise the nearest ktx.yaml or current working directory.\n', - ); + ) + .action(async (_options: unknown, command) => { + await runConnectionArgs(context, { command: 'list', projectDir: resolveCommandProjectDir(command) }); + }); connection.hook('preAction', (_thisCommand, actionCommand) => { context.writeDebug?.(commandName, actionCommand); }); @@ -32,25 +36,22 @@ export function registerConnectionCommands(program: Command, context: KtxCliComm connection .command('test') - .description('Test a configured connection') - .argument('[connectionId]', 'KTX connection id (omit when --all is set)') + .description('Test one or all configured connections (default: all)') + .argument('[connectionId]', 'KTX connection id to test (omit to test all)') .option('--all', 'Test every configured connection and print a summary list') .action(async (connectionId: string | undefined, options: { all?: boolean }, command) => { - const all = options.all === true; - if (all && connectionId !== undefined) { + if (options.all === true && connectionId !== undefined) { command.error('error: --all cannot be combined with a connection id argument'); } - if (!all && connectionId === undefined) { - command.error('error: missing required argument (or pass --all)'); - } - if (all) { + const selection = resolveConnectionSelection({ connectionId, all: options.all === true }); + if (selection.kind === 'all') { await runConnectionArgs(context, { command: 'test-all', projectDir: resolveCommandProjectDir(command) }); return; } await runConnectionArgs(context, { command: 'test', projectDir: resolveCommandProjectDir(command), - connectionId: connectionId as string, + connectionId: selection.connectionId, }); }); } diff --git a/packages/cli/src/commands/connection-selection.ts b/packages/cli/src/commands/connection-selection.ts new file mode 100644 index 00000000..5ea0ca46 --- /dev/null +++ b/packages/cli/src/commands/connection-selection.ts @@ -0,0 +1,18 @@ +export type ConnectionSelection = + | { kind: 'all' } + | { kind: 'single'; connectionId: string }; + +export interface ResolveConnectionSelectionInput { + connectionId?: string | undefined; + all: boolean; +} + +export function resolveConnectionSelection(input: ResolveConnectionSelectionInput): ConnectionSelection { + if (input.all && input.connectionId !== undefined) { + throw new Error('--all cannot be combined with a connection id argument'); + } + if (input.connectionId !== undefined) { + return { kind: 'single', connectionId: input.connectionId }; + } + return { kind: 'all' }; +} diff --git a/packages/cli/src/commands/ingest-commands.ts b/packages/cli/src/commands/ingest-commands.ts index 8e3bd7f2..9ffd2562 100644 --- a/packages/cli/src/commands/ingest-commands.ts +++ b/packages/cli/src/commands/ingest-commands.ts @@ -10,6 +10,7 @@ import { runtimeInstallPolicyFromFlags } from '../managed-python-command.js'; import type { KtxPublicIngestArgs } from '../public-ingest.js'; import { profileMark } from '../startup-profile.js'; import type { KtxTextIngestArgs } from '../text-ingest.js'; +import { resolveConnectionSelection } from './connection-selection.js'; profileMark('module:commands/ingest-commands'); @@ -24,15 +25,20 @@ export function registerIngestCommands( ): void { const ingest = program .command('ingest') - .description('Build or inspect KTX context') + .description('Build or inspect KTX context, or capture text into memory') .usage('[options] [connectionId]') - .argument('[connectionId]', 'Configured connection id to ingest') + .argument('[connectionId]', 'Configured connection id to ingest (omit to ingest all)') .option('--all', 'Ingest all configured connections', false) .addOption(new Option('--fast', 'Use deterministic database schema ingest').conflicts('deep')) .addOption(new Option('--deep', 'Use AI-enriched database ingest').conflicts('fast')) .addOption(new Option('--query-history', 'Include database query-history usage patterns').conflicts('noQueryHistory')) .addOption(new Option('--no-query-history', 'Skip database query-history usage patterns')) .option('--query-history-window-days ', 'Query-history lookback window for this run', parsePositiveIntegerOption) + .option('--text ', 'Capture inline text into KTX memory; repeatable', collectOption, []) + .option('--file ', 'Capture a text file into KTX memory; use - for stdin; repeatable', collectOption, []) + .option('--connection-id ', 'KTX connection id to tag captured text/file notes') + .option('--user-id ', 'Memory user id for text/file capture attribution', 'local-cli') + .option('--fail-fast', 'Stop after the first failed text/file item', false) .addOption(new Option('--plain', 'Print plain text output').conflicts(['json'])) .addOption(new Option('--json', 'Print JSON output').conflicts(['plain'])) .option('--yes', 'Install required managed runtime features without prompting') @@ -40,14 +46,45 @@ export function registerIngestCommands( .showHelpAfterError(); ingest.action(async (connectionId: string | undefined, options, command) => { + const projectDir = resolveCommandProjectDir(command); + const hasTextCapture = options.text.length > 0 || options.file.length > 0; + + if (hasTextCapture) { + if (connectionId !== undefined) { + command.error( + 'error: --text/--file does not accept a positional connection id; use --connection-id to tag captured notes', + ); + } + if (options.all === true) { + command.error('error: --all cannot be combined with --text or --file'); + } + context.setExitCode( + await commandOptions.runTextIngest( + { + projectDir, + texts: options.text, + files: options.file, + ...(options.connectionId ? { connectionId: options.connectionId } : {}), + userId: options.userId, + json: options.json === true, + failFast: options.failFast === true, + }, + context.io, + context.deps, + ), + ); + return; + } + + const selection = resolveConnectionSelection({ connectionId, all: options.all === true }); const { runKtxPublicIngest } = await import('../public-ingest.js'); const queryHistory = options.queryHistory === true ? 'enabled' : options.queryHistory === false ? 'disabled' : 'default'; const args: KtxPublicIngestArgs = { command: 'run', - projectDir: resolveCommandProjectDir(command), - ...(connectionId ? { targetConnectionId: connectionId } : {}), - all: options.all === true, + projectDir, + ...(selection.kind === 'single' ? { targetConnectionId: selection.connectionId } : {}), + all: selection.kind === 'all', json: options.json === true, inputMode: options.input === false ? 'disabled' : 'auto', ...(options.fast === true ? { depth: 'fast' as const } : {}), @@ -63,32 +100,4 @@ export function registerIngestCommands( ingest.hook('preAction', (_thisCommand, actionCommand) => { context.writeDebug?.('ingest', actionCommand); }); - - ingest - .command('text') - .description('Ingest free-form text artifacts into KTX memory') - .argument('[files...]', 'Files to ingest; use - to read one item from stdin') - .option('--text ', 'Text content to ingest; repeat for a batch', collectOption, []) - .option('--connection-id ', 'Optional KTX connection id for semantic-layer capture') - .option('--user-id ', 'Memory user id for capture attribution', 'local-cli') - .option('--json', 'Print JSON output') - .option('--fail-fast', 'Stop after the first failed text item', false) - .action(async (files: string[], options, command) => { - const parentOptions = command.parent?.opts() as { json?: boolean } | undefined; - context.setExitCode( - await commandOptions.runTextIngest( - { - projectDir: resolveCommandProjectDir(command), - texts: options.text, - files, - ...(options.connectionId ? { connectionId: options.connectionId } : {}), - userId: options.userId, - json: options.json === true || parentOptions?.json === true, - failFast: options.failFast === true, - }, - context.io, - context.deps, - ), - ); - }); } diff --git a/packages/cli/src/commands/knowledge-commands.ts b/packages/cli/src/commands/knowledge-commands.ts index 1c61a836..c0fe4f06 100644 --- a/packages/cli/src/commands/knowledge-commands.ts +++ b/packages/cli/src/commands/knowledge-commands.ts @@ -21,59 +21,29 @@ function isDebugEnabled(command: CommandWithGlobalOptions): boolean { } export function registerWikiCommands(program: Command, context: KtxCliCommandContext): void { - const wiki = program + program .command('wiki') .description('List or search local wiki pages') + .usage('[options] [query...]') + .argument('[query...]', 'Search query; omit to list all pages') + .option('--user-id ', 'Local user id', 'local') + .option('--limit ', 'Maximum search results (search mode only)', parsePositiveIntegerOption) + .addOption( + new Option('--output ', 'Output mode: pretty (default in TTY), plain (TSV), or json').choices([ + 'pretty', + 'plain', + 'json', + ]), + ) + .option('--json', 'Shortcut for --output=json (overrides --output)', false) .showHelpAfterError() .addHelpText( 'after', '\nProject directory defaults to KTX_PROJECT_DIR when set, otherwise the current working directory.\n', - ); - - wiki - .command('list') - .description('List local wiki pages') - .option('--user-id ', 'Local user id', 'local') - .addOption( - new Option('--output ', 'Output mode: pretty (default in TTY), plain (TSV), or json').choices([ - 'pretty', - 'plain', - 'json', - ]), ) - .option('--json', 'Shortcut for --output=json (overrides --output)', false) .action( async ( - options: { userId: string; output?: 'pretty' | 'plain' | 'json'; json?: boolean }, - command, - ) => { - await runKnowledgeArgs(context, { - command: 'list', - projectDir: resolveCommandProjectDir(command), - userId: options.userId, - output: options.output, - json: options.json, - }); - }, - ); - - wiki - .command('search') - .description('Search local wiki pages') - .argument('', 'Search query') - .option('--user-id ', 'Local user id', 'local') - .option('--limit ', 'Maximum search results', parsePositiveIntegerOption) - .addOption( - new Option('--output ', 'Output mode: pretty (default in TTY), plain (TSV), or json').choices([ - 'pretty', - 'plain', - 'json', - ]), - ) - .option('--json', 'Shortcut for --output=json (overrides --output)', false) - .action( - async ( - query: string, + query: string[], options: { userId: string; limit?: number; @@ -82,10 +52,20 @@ export function registerWikiCommands(program: Command, context: KtxCliCommandCon }, command, ) => { + if (query.length === 0) { + await runKnowledgeArgs(context, { + command: 'list', + projectDir: resolveCommandProjectDir(command), + userId: options.userId, + output: options.output, + json: options.json, + }); + return; + } await runKnowledgeArgs(context, { command: 'search', projectDir: resolveCommandProjectDir(command), - query, + query: query.join(' '), userId: options.userId, output: options.output, json: options.json, diff --git a/packages/cli/src/commands/mcp-commands.ts b/packages/cli/src/commands/mcp-commands.ts index be7044a7..94b17498 100644 --- a/packages/cli/src/commands/mcp-commands.ts +++ b/packages/cli/src/commands/mcp-commands.ts @@ -36,8 +36,24 @@ function formatMcpStartResultMessage(input: { status: 'started' | 'already-runni ].join('\n'); } +async function printMcpStatus(context: KtxCliCommandContext, projectDir: string): Promise { + const status = await (context.deps.mcp?.readStatus ?? readKtxMcpDaemonStatus)({ projectDir }); + context.io.stdout.write(`${status.detail}\n`); + if (status.kind === 'running') { + context.io.stdout.write(`URL: ${status.url}\n`); + context.io.stdout.write(`PID: ${status.state.pid}\n`); + context.io.stdout.write(`Token auth: ${status.state.tokenAuth ? 'enabled' : 'disabled'}\n`); + context.io.stdout.write(`Project: ${status.state.projectDir}\n`); + } +} + export function registerMcpCommands(program: Command, context: KtxCliCommandContext): void { - const mcp = program.command('mcp').description('Run the KTX MCP HTTP server'); + const mcp = program + .command('mcp') + .description('Manage the KTX MCP HTTP server (bare command: show status)') + .action(async (_options, command) => { + await printMcpStatus(context, resolveCommandProjectDir(command)); + }); mcp .command('stdio') @@ -110,16 +126,7 @@ export function registerMcpCommands(program: Command, context: KtxCliCommandCont .command('status') .description('Show KTX MCP daemon status') .action(async (_options, command) => { - const status = await (context.deps.mcp?.readStatus ?? readKtxMcpDaemonStatus)({ - projectDir: resolveCommandProjectDir(command), - }); - context.io.stdout.write(`${status.detail}\n`); - if (status.kind === 'running') { - context.io.stdout.write(`URL: ${status.url}\n`); - context.io.stdout.write(`PID: ${status.state.pid}\n`); - context.io.stdout.write(`Token auth: ${status.state.tokenAuth ? 'enabled' : 'disabled'}\n`); - context.io.stdout.write(`Project: ${status.state.projectDir}\n`); - } + await printMcpStatus(context, resolveCommandProjectDir(command)); }); mcp diff --git a/packages/cli/src/commands/sl-commands.ts b/packages/cli/src/commands/sl-commands.ts index d23674cd..6a03eb67 100644 --- a/packages/cli/src/commands/sl-commands.ts +++ b/packages/cli/src/commands/sl-commands.ts @@ -42,59 +42,49 @@ export function registerSlCommands(program: Command, context: KtxCliCommandConte const sl = program .command(commandName) .description('List, search, validate, or query local semantic-layer sources') + .usage('[options] [query...]') + .argument('[query...]', 'Search query; omit to list all sources') + .option('--connection-id ', 'KTX connection id') + .option('--limit ', 'Maximum search results (search mode only)', parsePositiveIntegerOption) + .addOption( + new Option('--output ', 'Output mode: pretty (default in TTY), plain (TSV), or json').choices([ + 'pretty', + 'plain', + 'json', + ]), + ) + .option('--json', 'Shortcut for --output=json (overrides --output)', false) .showHelpAfterError() .addHelpText( 'after', '\nProject directory defaults to KTX_PROJECT_DIR when set, otherwise the current working directory.\n', - ); - - sl.command('list') - .description('List semantic-layer sources') - .option('--connection-id ', 'KTX connection id') - .addOption( - new Option('--output ', 'Output mode: pretty (default in TTY), plain (TSV), or json').choices([ - 'pretty', - 'plain', - 'json', - ]), ) - .option('--json', 'Shortcut for --output=json (overrides --output)', false) - .action( - async (options: { connectionId?: string; output?: 'pretty' | 'plain' | 'json'; json?: boolean }, command) => { - await runSlArgs(context, { - command: 'list', - projectDir: resolveCommandProjectDir(command), - connectionId: options.connectionId, - output: options.output, - json: options.json, - }); - }, - ); - - sl.command('search') - .description('Search semantic-layer sources') - .argument('', 'Search query') - .option('--connection-id ', 'KTX connection id') - .option('--limit ', 'Maximum search results', parsePositiveIntegerOption) - .addOption( - new Option('--output ', 'Output mode: pretty (default in TTY), plain (TSV), or json').choices([ - 'pretty', - 'plain', - 'json', - ]), - ) - .option('--json', 'Shortcut for --output=json (overrides --output)', false) .action( async ( - query: string, - options: { connectionId?: string; limit?: number; output?: 'pretty' | 'plain' | 'json'; json?: boolean }, + query: string[], + options: { + connectionId?: string; + limit?: number; + output?: 'pretty' | 'plain' | 'json'; + json?: boolean; + }, command, ) => { + if (query.length === 0) { + await runSlArgs(context, { + command: 'list', + projectDir: resolveCommandProjectDir(command), + connectionId: options.connectionId, + output: options.output, + json: options.json, + }); + return; + } await runSlArgs(context, { command: 'search', projectDir: resolveCommandProjectDir(command), connectionId: options.connectionId, - query, + query: query.join(' '), ...(options.limit !== undefined ? { limit: options.limit } : {}), output: options.output, json: options.json, @@ -103,21 +93,24 @@ export function registerSlCommands(program: Command, context: KtxCliCommandConte ); sl.command('validate') - .description('Validate a semantic-layer source') + .description('Validate a semantic-layer source (set --connection-id on `ktx sl`)') .argument('', 'Semantic-layer source name') - .requiredOption('--connection-id ', 'KTX connection id') - .action(async (sourceName: string, options: { connectionId: string }, command) => { + .action(async (sourceName: string, _options, command) => { + const parentOpts = command.parent?.opts() as { connectionId?: string } | undefined; + const connectionId = parentOpts?.connectionId; + if (connectionId === undefined) { + command.error("error: required option '--connection-id ' not specified"); + } await runSlArgs(context, { command: 'validate', projectDir: resolveCommandProjectDir(command), - connectionId: options.connectionId, + connectionId: connectionId as string, sourceName, }); }); sl.command('query') - .description('Compile or execute a semantic-layer query') - .option('--connection-id ', 'KTX connection id') + .description('Compile or execute a semantic-layer query (set --connection-id on `ktx sl`)') .option('--query-file ', 'JSON semantic-layer query file') .option('--measure ', 'Measure to query; repeatable', collectOption, []) .option('--dimension ', 'Dimension to include; repeatable', collectOption, []) @@ -135,10 +128,11 @@ export function registerSlCommands(program: Command, context: KtxCliCommandConte if (options.measure.length === 0 && !options.queryFile) { throw new Error('sl query requires at least one --measure'); } + const parentOpts = command.parent?.opts() as { connectionId?: string } | undefined; const args = slQueryCommandSchema.parse({ command: 'query', projectDir: resolveCommandProjectDir(command), - connectionId: options.connectionId, + connectionId: parentOpts?.connectionId, ...(options.queryFile ? { queryFile: options.queryFile } : { diff --git a/packages/cli/src/doctor.test.ts b/packages/cli/src/doctor.test.ts index 2a397653..a6bfbff6 100644 --- a/packages/cli/src/doctor.test.ts +++ b/packages/cli/src/doctor.test.ts @@ -65,8 +65,8 @@ describe('formatDoctorReport', () => { expect(output).not.toContain('v22.16.0'); expect(output).toContain('Everything ready.'); expect(output).toContain('ktx status --json'); - expect(output).toContain('ktx sl list'); - expect(output).toContain('ktx wiki list'); + expect(output).toContain('ktx sl'); + expect(output).toContain('ktx wiki'); expect(output).not.toContain('ktx scan'); expect(output).not.toContain('ktx sl ask'); }); @@ -561,8 +561,8 @@ describe('runKtxDoctor', () => { expect(out).toContain('info: pg_stat_statements.max is 1000'); expect(out).not.toContain('Update the Postgres parameter group or config'); expect(out).toContain('ktx status --json'); - expect(out).toContain('ktx sl list'); - expect(out).toContain('ktx wiki list'); + expect(out).toContain('ktx sl'); + expect(out).toContain('ktx wiki'); expect(out).not.toContain('ktx scan'); expect(out).not.toContain('ktx sl ask'); delete process.env.ANTHROPIC_API_KEY; diff --git a/packages/cli/src/example-smoke.test.ts b/packages/cli/src/example-smoke.test.ts index e59d7d7e..22f1eafa 100644 --- a/packages/cli/src/example-smoke.test.ts +++ b/packages/cli/src/example-smoke.test.ts @@ -72,13 +72,13 @@ describe('standalone local warehouse example', () => { it('runs local CLI commands against the copied example project', async () => { const projectDir = await copyExampleProject(tempDir); - const knowledgeList = await runBuiltCli(['wiki', 'search', 'revenue', '--json', '--project-dir', projectDir]); + const knowledgeList = await runBuiltCli(['wiki', 'revenue', '--json', '--project-dir', projectDir]); expect(knowledgeList).toMatchObject({ code: 0, stderr: '' }); expect( parseJsonOutput<{ data: { items: Array<{ key: string; summary: string }> } }>(knowledgeList.stdout).data.items, ).toContainEqual(expect.objectContaining({ key: 'revenue', summary: 'Paid order value after refunds' })); - const slList = await runBuiltCli(['sl', 'list', '--json', '--project-dir', projectDir, '--connection-id', 'warehouse']); + const slList = await runBuiltCli(['sl', '--json', '--project-dir', projectDir, '--connection-id', 'warehouse']); expect(slList).toMatchObject({ code: 0, stderr: '' }); expect( parseJsonOutput<{ data: { items: Array<{ connectionId: string; name: string; columnCount: number }> } }>( @@ -110,7 +110,7 @@ describe('standalone local warehouse example', () => { 'fake', ]); expect(ingest).toMatchObject({ code: 1, stdout: '' }); - expect(ingest.stderr).toContain("unknown option '--connection-id'"); + expect(ingest.stderr).toContain("unknown option '--adapter'"); }, 30_000); }); diff --git a/packages/cli/src/index.test.ts b/packages/cli/src/index.test.ts index 3c12e583..9a7fceeb 100644 --- a/packages/cli/src/index.test.ts +++ b/packages/cli/src/index.test.ts @@ -129,9 +129,10 @@ describe('runKtxCli', () => { expect(testIo.stdout()).toContain('Usage: ktx [options] [command]'); expect(testIo.stdout()).toContain('KTX data agent context layer CLI'); - for (const command of ['setup', 'connection', 'ingest', 'wiki', 'sl', 'status', 'dev']) { + for (const command of ['setup', 'connection', 'ingest', 'wiki', 'sl', 'status', 'admin']) { expect(testIo.stdout()).toContain(`${command}`); } + expect(testIo.stdout()).not.toMatch(/^ dev\s/m); expect(testIo.stdout()).not.toMatch(/^ scan\s/m); for (const removed of ['demo', 'init', 'connect', 'ask', 'knowledge', 'agent', 'completion', 'serve']) { expect(testIo.stdout()).not.toMatch(new RegExp(`^\\s+${removed}(?:\\s|\\[|$)`, 'm')); @@ -148,7 +149,7 @@ describe('runKtxCli', () => { const knowledge = vi.fn(async () => 0); const listIo = makeIo(); - await expect(runKtxCli(['--project-dir', tempDir, 'wiki', 'list', '--json'], listIo.io, { knowledge })) + await expect(runKtxCli(['--project-dir', tempDir, 'wiki', '--json'], listIo.io, { knowledge })) .resolves.toBe(0); expect(knowledge).toHaveBeenCalledWith( { @@ -162,7 +163,7 @@ describe('runKtxCli', () => { const searchIo = makeIo(); await expect( - runKtxCli(['--project-dir', tempDir, 'wiki', 'search', 'revenue', '--limit', '5'], searchIo.io, { knowledge }), + runKtxCli(['--project-dir', tempDir, 'wiki', 'revenue', '--limit', '5'], searchIo.io, { knowledge }), ).resolves.toBe(0); expect(knowledge).toHaveBeenLastCalledWith( { @@ -178,7 +179,7 @@ describe('runKtxCli', () => { const debugSearchIo = makeIo(); await expect( - runKtxCli(['--project-dir', tempDir, '--debug', 'wiki', 'search', 'revenue'], debugSearchIo.io, { knowledge }), + runKtxCli(['--project-dir', tempDir, '--debug', 'wiki', 'revenue'], debugSearchIo.io, { knowledge }), ).resolves.toBe(0); expect(knowledge).toHaveBeenLastCalledWith( { @@ -191,47 +192,57 @@ describe('runKtxCli', () => { }, debugSearchIo.io, ); + + const multiWordIo = makeIo(); + await expect( + runKtxCli(['--project-dir', tempDir, 'wiki', 'revenue', 'policy'], multiWordIo.io, { knowledge }), + ).resolves.toBe(0); + expect(knowledge).toHaveBeenLastCalledWith( + { + command: 'search', + projectDir: tempDir, + query: 'revenue policy', + userId: 'local', + json: false, + }, + multiWordIo.io, + ); }); - it('rejects removed public wiki read and write commands', async () => { + it('rejects unknown write-style flags on the flattened wiki and sl commands', async () => { const knowledge = vi.fn(async () => 0); - - for (const argv of [ - ['--project-dir', tempDir, 'wiki', 'read', 'revenue', '--json'], - ['--project-dir', tempDir, 'wiki', 'write', 'revenue', '--summary', 'Revenue', '--content', 'Revenue.'], - ]) { - const io = makeIo(); - - await expect(runKtxCli(argv, io.io, { knowledge })).resolves.toBe(1); - - expect(io.stderr()).toMatch(/unknown command|error:/); - } - - expect(knowledge).not.toHaveBeenCalled(); - }); - - it('rejects removed public sl read/write commands', async () => { const sl = vi.fn(async () => 0); - for (const argv of [ - ['--project-dir', tempDir, 'sl', 'read', 'orders', '--connection-id', 'warehouse'], - ['--project-dir', tempDir, 'sl', 'write', 'orders', '--connection-id', 'warehouse', '--yaml', 'name: orders'], - ]) { - const io = makeIo(); - await expect(runKtxCli(argv, io.io, { sl })).resolves.toBe(1); - expect(io.stderr()).toMatch(/unknown command|error:/); - } + const wikiIo = makeIo(); + await expect( + runKtxCli( + ['--project-dir', tempDir, 'wiki', 'revenue', '--summary', 'Revenue', '--content', 'Revenue.'], + wikiIo.io, + { knowledge }, + ), + ).resolves.toBe(1); + expect(wikiIo.stderr()).toMatch(/unknown option|error:/); + expect(knowledge).not.toHaveBeenCalled(); + const slIo = makeIo(); + await expect( + runKtxCli( + ['--project-dir', tempDir, 'sl', 'orders', '--yaml', 'name: orders'], + slIo.io, + { sl }, + ), + ).resolves.toBe(1); + expect(slIo.stderr()).toMatch(/unknown option|error:/); expect(sl).not.toHaveBeenCalled(); }); - it('routes sl search and rejects the old sl list --query flag', async () => { + it('routes sl search via the flattened query positional and rejects unknown flags', async () => { const sl = vi.fn(async () => 0); const searchIo = makeIo(); await expect( runKtxCli( - ['--project-dir', tempDir, 'sl', 'search', 'revenue', '--connection-id', 'warehouse', '--limit', '5', '--json'], + ['--project-dir', tempDir, 'sl', 'revenue', '--connection-id', 'warehouse', '--limit', '5', '--json'], searchIo.io, { sl }, ), @@ -249,11 +260,26 @@ describe('runKtxCli', () => { searchIo.io, ); - const listIo = makeIo(); + const bareIo = makeIo(); await expect( - runKtxCli(['--project-dir', tempDir, 'sl', 'list', '--query', 'revenue'], listIo.io, { sl }), + runKtxCli(['--project-dir', tempDir, 'sl', '--connection-id', 'warehouse', '--json'], bareIo.io, { sl }), + ).resolves.toBe(0); + expect(sl).toHaveBeenLastCalledWith( + { + command: 'list', + projectDir: tempDir, + connectionId: 'warehouse', + json: true, + output: undefined, + }, + bareIo.io, + ); + + const unknownIo = makeIo(); + await expect( + runKtxCli(['--project-dir', tempDir, 'sl', '--query', 'revenue'], unknownIo.io, { sl }), ).resolves.toBe(1); - expect(listIo.stderr()).toContain("unknown option '--query'"); + expect(unknownIo.stderr()).toContain("unknown option '--query'"); }); it('routes runtime management commands with the release runtime version', async () => { @@ -266,17 +292,17 @@ describe('runKtxCli', () => { const pruneIo = makeIo(); await expect( - runKtxCli(['dev', 'runtime', 'install', '--feature', 'local-embeddings', '--force', '--yes'], installIo.io, { + runKtxCli(['admin', 'runtime', 'install', '--feature', 'local-embeddings', '--force', '--yes'], installIo.io, { runtime, }), ).resolves.toBe(0); await expect( - runKtxCli(['dev', 'runtime', 'start', '--feature', 'local-embeddings', '--force'], startIo.io, { runtime }), + runKtxCli(['admin', 'runtime', 'start', '--feature', 'local-embeddings', '--force'], startIo.io, { runtime }), ).resolves.toBe(0); - await expect(runKtxCli(['dev', 'runtime', 'stop'], stopIo.io, { runtime })).resolves.toBe(0); - await expect(runKtxCli(['dev', 'runtime', 'stop', '--all'], stopAllIo.io, { runtime })).resolves.toBe(0); - await expect(runKtxCli(['dev', 'runtime', 'status', '--json'], statusIo.io, { runtime })).resolves.toBe(0); - await expect(runKtxCli(['dev', 'runtime', 'prune', '--dry-run'], pruneIo.io, { runtime })).resolves.toBe(1); + await expect(runKtxCli(['admin', 'runtime', 'stop'], stopIo.io, { runtime })).resolves.toBe(0); + await expect(runKtxCli(['admin', 'runtime', 'stop', '--all'], stopAllIo.io, { runtime })).resolves.toBe(0); + await expect(runKtxCli(['admin', 'runtime', 'status', '--json'], statusIo.io, { runtime })).resolves.toBe(0); + await expect(runKtxCli(['admin', 'runtime', 'prune', '--dry-run'], pruneIo.io, { runtime })).resolves.toBe(1); expect(runtime).toHaveBeenNthCalledWith( 1, @@ -377,7 +403,7 @@ describe('runKtxCli', () => { it('documents runtime stop all in command help', async () => { const testIo = makeIo(); - await expect(runKtxCli(['dev', 'runtime', 'stop', '--help'], testIo.io)).resolves.toBe(0); + await expect(runKtxCli(['admin', 'runtime', 'stop', '--help'], testIo.io)).resolves.toBe(0); expect(testIo.stdout()).toContain('--all'); expect(testIo.stdout()).toContain('Stop all KTX daemon processes recorded or discoverable'); @@ -523,7 +549,7 @@ describe('runKtxCli', () => { await initKtxProject({ projectDir }); const commands = [ ['--project-dir', projectDir, 'status', '--json'], - ['--project-dir', projectDir, 'sl', 'list', '--json'], + ['--project-dir', projectDir, 'sl', '--json'], ]; for (const argv of commands) { @@ -655,9 +681,9 @@ describe('runKtxCli', () => { const completionIo = makeIo(); const hiddenIo = makeIo(); - await expect(runKtxCli(['dev', 'completion', 'zsh'], completionIo.io)).resolves.toBe(1); + await expect(runKtxCli(['admin', 'completion', 'zsh'], completionIo.io)).resolves.toBe(1); await expect( - runKtxCli(['dev', '__complete', '--shell', 'zsh', '--position', '2', '--', 'ktx', 'co'], hiddenIo.io), + runKtxCli(['admin', '__complete', '--shell', 'zsh', '--position', '2', '--', 'ktx', 'co'], hiddenIo.io), ).resolves.toBe(1); expect(completionIo.stderr()).toMatch(/unknown command|error:/); @@ -871,7 +897,8 @@ describe('runKtxCli', () => { expect(testIo.stdout()).toContain('--query-history'); expect(testIo.stdout()).toContain('--no-query-history'); expect(testIo.stdout()).toContain('--query-history-window-days '); - expect(testIo.stdout()).toContain('text'); + expect(testIo.stdout()).toContain('--text'); + expect(testIo.stdout()).toContain('--file'); expect(testIo.stdout()).not.toMatch(/^ status\s/m); expect(testIo.stdout()).not.toMatch(/^ replay\s/m); expect(testIo.stdout()).not.toMatch(/^ run\s/m); @@ -891,7 +918,6 @@ describe('runKtxCli', () => { '--project-dir', tempDir, 'ingest', - 'text', '--text', 'Revenue means gross receipts.', '--text', @@ -923,22 +949,45 @@ describe('runKtxCli', () => { expect(testIo.stderr()).toBe(''); }); - it('documents text ingest inputs without a manifest option', async () => { + it('rejects a positional connection id when --text is supplied', async () => { const textIngest = vi.fn(async () => 0); + const publicIngest = vi.fn(async () => 0); const testIo = makeIo(); - await expect(runKtxCli(['ingest', 'text', '--help'], testIo.io, { textIngest })).resolves.toBe(0); + await expect( + runKtxCli( + ['--project-dir', tempDir, 'ingest', 'warehouse', '--text', 'hello'], + testIo.io, + { textIngest, publicIngest }, + ), + ).resolves.toBe(1); - expect(testIo.stdout()).toContain('Usage: ktx ingest text [options] [files...]'); - expect(testIo.stdout()).toContain('--text '); - expect(testIo.stdout()).toContain('--connection-id '); - expect(testIo.stdout()).toContain('--user-id '); - expect(testIo.stdout()).toContain('--fail-fast'); - expect(testIo.stdout()).not.toContain('--manifest'); expect(textIngest).not.toHaveBeenCalled(); + expect(publicIngest).not.toHaveBeenCalled(); + expect(testIo.stderr()).toMatch(/--text\/--file does not accept a positional connection id/); }); - it('rejects old adapter-backed ingest flags at the top level and under dev', async () => { + it('treats bare ingest as ingest --all', async () => { + const publicIngest = vi.fn().mockResolvedValue(0); + const testIo = makeIo(); + + await expect( + runKtxCli(['--project-dir', tempDir, 'ingest', '--no-input'], testIo.io, { publicIngest }), + ).resolves.toBe(0); + + expect(publicIngest).toHaveBeenCalledWith( + expect.objectContaining({ + command: 'run', + projectDir: tempDir, + all: true, + }), + testIo.io, + ); + const args = publicIngest.mock.calls[0]?.[0] as { targetConnectionId?: string }; + expect(args.targetConnectionId).toBeUndefined(); + }); + + it('rejects old adapter-backed ingest flags at the top level and under admin', async () => { const rootRunIo = makeIo(); const devRunIo = makeIo(); const publicIngest = vi.fn(async () => 0); @@ -949,7 +998,7 @@ describe('runKtxCli', () => { }), ).resolves.toBe(1); await expect( - runKtxCli(['dev', 'ingest', 'run', '--connection-id', 'warehouse', '--adapter', 'metabase'], devRunIo.io, { + runKtxCli(['admin', 'ingest', 'run', '--connection-id', 'warehouse', '--adapter', 'metabase'], devRunIo.io, { publicIngest, }), ).resolves.toBe(1); @@ -958,12 +1007,12 @@ describe('runKtxCli', () => { expect(devRunIo.stderr()).toMatch(/unknown command|error:/); }); - it('rejects removed dev doctor and removed ingest parser cases', async () => { + it('rejects removed admin doctor and removed ingest parser cases', async () => { const doctor = vi.fn(async () => 0); const doctorIo = makeIo(); const ingestRunIo = makeIo(); - await expect(runKtxCli(['dev', 'doctor', 'setup', '--json', '--no-input'], doctorIo.io, { doctor })).resolves.toBe(1); + await expect(runKtxCli(['admin', 'doctor', 'setup', '--json', '--no-input'], doctorIo.io, { doctor })).resolves.toBe(1); await expect( runKtxCli( [ @@ -1755,12 +1804,12 @@ describe('runKtxCli', () => { expect(serveIo.stderr()).toMatch(/unknown command|error:/); }); - it('prints dev help for bare dev commands', async () => { + it('prints admin help for bare admin commands', async () => { const testIo = makeIo(); - await expect(runKtxCli(['dev'], testIo.io)).resolves.toBe(0); + await expect(runKtxCli(['admin'], testIo.io)).resolves.toBe(0); - expect(testIo.stdout()).toContain('Usage: ktx dev [options] [command]'); + expect(testIo.stdout()).toContain('Usage: ktx admin [options] [command]'); expect(testIo.stdout()).toContain('Low-level project initialization'); expect(testIo.stdout()).toContain('init'); expect(testIo.stdout()).toContain('runtime'); @@ -1772,13 +1821,13 @@ describe('runKtxCli', () => { expect(testIo.stderr()).toBe(''); }); - it('rejects removed dev command groups without invoking execution', async () => { + it('rejects removed admin command groups without invoking execution', async () => { for (const command of ['scan', 'ingest', 'mapping']) { const testIo = makeIo(); const publicIngest = vi.fn().mockResolvedValue(0); const sl = vi.fn().mockResolvedValue(0); - await expect(runKtxCli(['dev', command], testIo.io, { publicIngest, sl })).resolves.toBe(1); + await expect(runKtxCli(['admin', command], testIo.io, { publicIngest, sl })).resolves.toBe(1); expect(testIo.stderr()).toMatch(/unknown command|error:/); expect(publicIngest).not.toHaveBeenCalled(); @@ -1786,10 +1835,10 @@ describe('runKtxCli', () => { } }); - it('rejects removed reserved dev subcommands', async () => { + it('rejects removed reserved admin subcommands', async () => { const testIo = makeIo(); - await expect(runKtxCli(['dev', 'artifacts'], testIo.io)).resolves.toBe(1); + await expect(runKtxCli(['admin', 'artifacts'], testIo.io)).resolves.toBe(1); expect(testIo.stderr()).toMatch(/unknown command|error:/); }); diff --git a/packages/cli/src/io/print-list.test.ts b/packages/cli/src/io/print-list.test.ts index 0cb5a537..f084e519 100644 --- a/packages/cli/src/io/print-list.test.ts +++ b/packages/cli/src/io/print-list.test.ts @@ -78,14 +78,14 @@ describe('printList — plain mode', () => { mode: 'plain', command: 'sl search', emptyMessage: 'No sources matched "foo"', - emptyHint: 'Run `ktx sl list` to see available sources.', + emptyHint: 'Run `ktx sl` to see available sources.', unit: 'source', io: r.io, }); expect(r.out()).toBe(''); expect(r.err()).toBe( 'No sources matched "foo"\n' + - 'Run `ktx sl list` to see available sources.\n', + 'Run `ktx sl` to see available sources.\n', ); }); }); @@ -188,13 +188,13 @@ describe('printList — pretty mode', () => { mode: 'pretty', command: 'sl search', emptyMessage: 'No sources matched "foo"', - emptyHint: 'Run `ktx sl list` to see available sources.', + emptyHint: 'Run `ktx sl` to see available sources.', unit: 'source', io: r.io, }); const out = stripAnsi(r.out()); expect(out).toContain('No sources matched "foo"'); - expect(out).toContain('Run `ktx sl list` to see available sources.'); + expect(out).toContain('Run `ktx sl` to see available sources.'); }); it('singularizes the footer when there is one row', () => { diff --git a/packages/cli/src/knowledge.ts b/packages/cli/src/knowledge.ts index 8213d05d..f12d3567 100644 --- a/packages/cli/src/knowledge.ts +++ b/packages/cli/src/knowledge.ts @@ -130,7 +130,7 @@ export async function runKtxKnowledge( } const mode = resolveOutputMode({ explicit: args.output, json: args.json, io }); let emptyMessage = `No local wiki pages matched "${args.query}"`; - let emptyHint = 'Run `ktx wiki list` to inspect available pages.'; + let emptyHint = 'Run `ktx wiki` to inspect available pages.'; if (results.length === 0 && mode !== 'json') { const pages = await listLocalKnowledgePages(project, { userId: args.userId }); if (pages.length === 0) { diff --git a/packages/cli/src/managed-python-command.test.ts b/packages/cli/src/managed-python-command.test.ts index 767d8dd1..717accf4 100644 --- a/packages/cli/src/managed-python-command.test.ts +++ b/packages/cli/src/managed-python-command.test.ts @@ -118,9 +118,9 @@ function makeSpinnerEvents() { describe('managedRuntimeInstallCommand', () => { it('prints the exact command for each managed runtime feature', () => { - expect(managedRuntimeInstallCommand('core')).toBe('ktx dev runtime install --yes'); + expect(managedRuntimeInstallCommand('core')).toBe('ktx admin runtime install --yes'); expect(managedRuntimeInstallCommand('local-embeddings')).toBe( - 'ktx dev runtime install --feature local-embeddings --yes', + 'ktx admin runtime install --feature local-embeddings --yes', ); }); }); @@ -221,7 +221,7 @@ describe('createManagedPythonSemanticLayerComputePort', () => { readStatus: vi.fn(async () => missingStatus()), installRuntime, }), - ).rejects.toThrow('KTX Python runtime is required for this command. Run: ktx dev runtime install --yes'); + ).rejects.toThrow('KTX Python runtime is required for this command. Run: ktx admin runtime install --yes'); expect(installRuntime).not.toHaveBeenCalled(); }); diff --git a/packages/cli/src/managed-python-command.ts b/packages/cli/src/managed-python-command.ts index 11e794ff..83953602 100644 --- a/packages/cli/src/managed-python-command.ts +++ b/packages/cli/src/managed-python-command.ts @@ -53,8 +53,8 @@ export interface ManagedPythonSemanticLayerComputeOptions extends ManagedPythonC export function managedRuntimeInstallCommand(feature: KtxRuntimeFeature): string { return feature === 'local-embeddings' - ? 'ktx dev runtime install --feature local-embeddings --yes' - : 'ktx dev runtime install --yes'; + ? 'ktx admin runtime install --feature local-embeddings --yes' + : 'ktx admin runtime install --yes'; } function installPrompt(feature: KtxRuntimeFeature): string { diff --git a/packages/cli/src/managed-python-runtime.test.ts b/packages/cli/src/managed-python-runtime.test.ts index 13b97a45..92e34e35 100644 --- a/packages/cli/src/managed-python-runtime.test.ts +++ b/packages/cli/src/managed-python-runtime.test.ts @@ -513,7 +513,7 @@ describe('doctorManagedPythonRuntime', () => { ['asset', 'pass'], ['runtime', 'fail'], ]); - expect(checks[2]?.fix).toBe('Run: ktx dev runtime install --yes'); + expect(checks[2]?.fix).toBe('Run: ktx admin runtime install --yes'); }); it('reports uv as a hard prerequisite when uv is missing', async () => { @@ -534,7 +534,7 @@ describe('doctorManagedPythonRuntime', () => { label: 'uv', status: 'fail', detail: MISSING_UV_RUNTIME_INSTALL_MESSAGE, - fix: 'Install uv, make sure it is on PATH, and run: ktx dev runtime install --yes', + fix: 'Install uv, make sure it is on PATH, and run: ktx admin runtime install --yes', }); }); }); diff --git a/packages/cli/src/managed-python-runtime.ts b/packages/cli/src/managed-python-runtime.ts index 88b0fa2b..68272840 100644 --- a/packages/cli/src/managed-python-runtime.ts +++ b/packages/cli/src/managed-python-runtime.ts @@ -122,7 +122,7 @@ export interface ManagedPythonRuntimeDoctorCheck { } export const MISSING_UV_RUNTIME_INSTALL_MESSAGE = - 'uv is required to install the KTX Python runtime. KTX does not download uv automatically. Install uv, make sure it is on PATH, and retry: ktx dev runtime install --yes'; + 'uv is required to install the KTX Python runtime. KTX does not download uv automatically. Install uv, make sure it is on PATH, and retry: ktx admin runtime install --yes'; function defaultAssetDir(): string { return fileURLToPath(new URL('../assets/python/', import.meta.url)); @@ -471,7 +471,7 @@ export async function doctorManagedPythonRuntime( id: 'uv', label: 'uv', detail: error instanceof Error ? error.message : String(error), - fix: 'Install uv, make sure it is on PATH, and run: ktx dev runtime install --yes', + fix: 'Install uv, make sure it is on PATH, and run: ktx admin runtime install --yes', }), ); } @@ -496,7 +496,7 @@ export async function doctorManagedPythonRuntime( id: 'runtime', label: 'Managed Python runtime', detail: status.detail, - ...(status.kind === 'ready' ? {} : { fix: 'Run: ktx dev runtime install --yes' }), + ...(status.kind === 'ready' ? {} : { fix: 'Run: ktx admin runtime install --yes' }), }), ); return checks; diff --git a/packages/cli/src/memory-flow-tui.test.tsx b/packages/cli/src/memory-flow-tui.test.tsx index e525a834..405fa18c 100644 --- a/packages/cli/src/memory-flow-tui.test.tsx +++ b/packages/cli/src/memory-flow-tui.test.tsx @@ -198,8 +198,8 @@ describe('MemoryFlowTuiApp', () => { expect(frame).toContain('order lifecycle'); expect(frame).toContain('customer metrics'); expect(frame).toContain('KTX finished ingesting your data'); - expect(frame).toContain('ktx sl list'); - expect(frame).toContain('ktx wiki list'); + expect(frame).toContain('ktx sl'); + expect(frame).toContain('ktx wiki'); expect(frame).not.toContain('ktx serve --mcp stdio --user-id local'); expect(frame).not.toContain(['ktx', 'ask'].join(' ')); expect(frame).not.toContain(['ktx', 'mcp'].join(' ')); diff --git a/packages/cli/src/next-steps.test.ts b/packages/cli/src/next-steps.test.ts index c2b15530..8a3e5e2a 100644 --- a/packages/cli/src/next-steps.test.ts +++ b/packages/cli/src/next-steps.test.ts @@ -10,8 +10,8 @@ describe('KTX demo next steps', () => { it('uses supported context-build commands before agent usage', () => { expect(KTX_CONTEXT_BUILD_COMMANDS).toEqual([ { - command: 'ktx ingest --all', - description: 'Build or refresh agent-ready context from configured connections', + command: 'ktx ingest', + description: 'Build or refresh agent-ready context from all configured connections', }, { command: 'ktx status', @@ -27,11 +27,11 @@ describe('KTX demo next steps', () => { description: 'Verify project setup and context readiness', }, { - command: 'ktx sl list', + command: 'ktx sl', description: 'Inspect generated semantic-layer sources', }, { - command: 'ktx wiki list', + command: 'ktx wiki', description: 'Inspect generated wiki pages', }, ]); @@ -67,7 +67,7 @@ describe('KTX demo next steps', () => { expect(rendered).toContain('Build KTX context next.'); expect(rendered).toContain('Run ingest to build database schema context before context-source ingest.'); - expect(rendered).toContain('ktx ingest --all'); + expect(rendered).toContain('ktx ingest'); expect(rendered).not.toContain('resume'); expect(rendered).not.toContain('scan'); expect(rendered).toContain('ktx status'); diff --git a/packages/cli/src/next-steps.ts b/packages/cli/src/next-steps.ts index 06ef3412..5410eee8 100644 --- a/packages/cli/src/next-steps.ts +++ b/packages/cli/src/next-steps.ts @@ -1,7 +1,7 @@ export const KTX_CONTEXT_BUILD_COMMANDS = [ { - command: 'ktx ingest --all', - description: 'Build or refresh agent-ready context from configured connections', + command: 'ktx ingest', + description: 'Build or refresh agent-ready context from all configured connections', }, { command: 'ktx status', @@ -15,11 +15,11 @@ export const KTX_NEXT_STEP_DIRECT_COMMANDS = [ description: 'Verify project setup and context readiness', }, { - command: 'ktx sl list', + command: 'ktx sl', description: 'Inspect generated semantic-layer sources', }, { - command: 'ktx wiki list', + command: 'ktx wiki', description: 'Inspect generated wiki pages', }, ] as const; diff --git a/packages/cli/src/print-command-tree.test.ts b/packages/cli/src/print-command-tree.test.ts index ececa88c..edd0b69a 100644 --- a/packages/cli/src/print-command-tree.test.ts +++ b/packages/cli/src/print-command-tree.test.ts @@ -12,7 +12,7 @@ describe('renderKtxCommandTree', () => { .filter((line) => /^ {2}[├└]── \S/.test(line)) .map((line) => line.replace(/^ {2}[├└]── /, '').trim().split(' ')[0]); - for (const expected of ['setup', 'connection', 'ingest', 'sl', 'mcp', 'dev']) { + for (const expected of ['setup', 'connection', 'ingest', 'sl', 'mcp', 'admin']) { expect(topLevel).toContain(expected); } diff --git a/packages/cli/src/public-ingest.test.ts b/packages/cli/src/public-ingest.test.ts index d7f853c8..47b80074 100644 --- a/packages/cli/src/public-ingest.test.ts +++ b/packages/cli/src/public-ingest.test.ts @@ -124,12 +124,15 @@ describe('buildPublicIngestPlan', () => { }); }); - it('rejects bare non-interactive ingest until the interactive confirmation slice exists', () => { - const project = projectWithConnections({ warehouse: { driver: 'postgres' } }); + it('treats a bare invocation (no connection id, no --all) as all configured connections', () => { + const project = projectWithConnections({ + warehouse: { driver: 'postgres' }, + docs: { driver: 'notion' }, + }); - expect(() => buildPublicIngestPlan(project, { projectDir: '/tmp/project', all: false })).toThrow( - 'Context build requires a connection id or all targets', - ); + const plan = buildPublicIngestPlan(project, { projectDir: '/tmp/project', all: false }); + + expect(plan.targets.map((target) => target.connectionId).sort()).toEqual(['docs', 'warehouse']); }); it('resolves database depth from flags, stored context, and defaults', () => { diff --git a/packages/cli/src/public-ingest.ts b/packages/cli/src/public-ingest.ts index 537dcec7..2c0a2856 100644 --- a/packages/cli/src/public-ingest.ts +++ b/packages/cli/src/public-ingest.ts @@ -469,14 +469,11 @@ export function buildPublicIngestPlan( scanMode?: Extract['mode']; }, ): KtxPublicIngestPlan { - if (!args.all && !args.targetConnectionId) { - throw new Error('Context build requires a connection id or all targets'); - } - + const allConnections = args.all || !args.targetConnectionId; const entries = Object.entries(project.config.connections).sort(([a], [b]) => a.localeCompare(b)); - const selected = args.all ? entries : entries.filter(([connectionId]) => connectionId === args.targetConnectionId); + const selected = allConnections ? entries : entries.filter(([connectionId]) => connectionId === args.targetConnectionId); - if (!args.all && selected.length === 0) { + if (!allConnections && selected.length === 0) { throw new Error(`Connection "${args.targetConnectionId}" is not configured in ktx.yaml`); } if (selected.length === 0) { diff --git a/packages/cli/src/runtime.test.ts b/packages/cli/src/runtime.test.ts index 01a529e7..e4a7883f 100644 --- a/packages/cli/src/runtime.test.ts +++ b/packages/cli/src/runtime.test.ts @@ -291,7 +291,7 @@ describe('runKtxRuntime', () => { label: 'Managed Python runtime', status: 'fail', detail: 'No runtime manifest at /runtime/0.2.0/manifest.json', - fix: 'Run: ktx dev runtime install --yes', + fix: 'Run: ktx admin runtime install --yes', }, ]), }; diff --git a/packages/cli/src/scan.test.ts b/packages/cli/src/scan.test.ts index b08c15de..8fa51130 100644 --- a/packages/cli/src/scan.test.ts +++ b/packages/cli/src/scan.test.ts @@ -368,8 +368,8 @@ describe('runKtxScan', () => { expect(io.stdout()).toContain('Report: raw-sources/warehouse/live-database/sync-1/scan-report.json'); expect(io.stdout()).toContain('Next:\n'); expect(io.stdout()).toContain('ktx status --project-dir '); - expect(io.stdout()).not.toContain('ktx dev scan status'); - expect(io.stdout()).not.toContain('ktx dev scan report'); + expect(io.stdout()).not.toContain('ktx admin scan status'); + expect(io.stdout()).not.toContain('ktx admin scan report'); expect(io.stdout()).not.toContain('\u001b['); expect(io.stdout()).not.toContain('✓'); expect(io.stdout()).not.toContain('+1'); diff --git a/packages/cli/src/setup-agents.test.ts b/packages/cli/src/setup-agents.test.ts index bd9c9458..e6ca39ed 100644 --- a/packages/cli/src/setup-agents.test.ts +++ b/packages/cli/src/setup-agents.test.ts @@ -169,7 +169,7 @@ describe('setup agents', () => { expect(skill).toContain(`--project-dir ${tempDir}`); expect(skill).toContain('must not print secrets'); expect(skill).toContain('status --json'); - expect(skill).toContain('sl list --json'); + expect(skill).toContain('sl --json'); expect(skill).toContain('sl query'); expect(skill).toContain('--format json'); expect(skill).not.toContain('sl query --json'); diff --git a/packages/cli/src/setup-agents.ts b/packages/cli/src/setup-agents.ts index 6fcd06c1..ecac2917 100644 --- a/packages/cli/src/setup-agents.ts +++ b/packages/cli/src/setup-agents.ts @@ -569,8 +569,8 @@ function cliInstructionContent(input: { projectDir: string; launcher: KtxCliLaun 'Available commands:', '', `- \`${ktxCommandLine(input.launcher, ['status', ...jsonProjectDirArgs])}\``, - `- \`${ktxCommandLine(input.launcher, ['sl', 'list', ...jsonProjectDirArgs])}\``, - `- \`${ktxCommandLine(input.launcher, ['sl', 'search', '', ...jsonProjectDirArgs, '--connection-id', ''])}\``, + `- \`${ktxCommandLine(input.launcher, ['sl', ...jsonProjectDirArgs])}\``, + `- \`${ktxCommandLine(input.launcher, ['sl', '', ...jsonProjectDirArgs, '--connection-id', ''])}\``, `- \`${ktxCommandLine(input.launcher, [ 'sl', 'query', @@ -585,7 +585,7 @@ function cliInstructionContent(input: { projectDir: string; launcher: KtxCliLaun '--max-rows', '100', ])}\``, - `- \`${ktxCommandLine(input.launcher, ['wiki', 'search', '', ...jsonProjectDirArgs, '--limit', '10'])}\``, + `- \`${ktxCommandLine(input.launcher, ['wiki', '', ...jsonProjectDirArgs, '--limit', '10'])}\``, '', 'Use semantic-layer queries before direct database access. Do not print secrets or credential references.', '', diff --git a/packages/cli/src/setup-embeddings.test.ts b/packages/cli/src/setup-embeddings.test.ts index 7e22be26..36df256a 100644 --- a/packages/cli/src/setup-embeddings.test.ts +++ b/packages/cli/src/setup-embeddings.test.ts @@ -286,7 +286,7 @@ describe('setup embeddings step', () => { const io = makeIo(); const ensureLocalEmbeddings = vi.fn(async () => { throw new Error( - 'KTX Python runtime is required for this command. Run: ktx dev runtime install --feature local-embeddings --yes', + 'KTX Python runtime is required for this command. Run: ktx admin runtime install --feature local-embeddings --yes', ); }); @@ -304,7 +304,7 @@ describe('setup embeddings step', () => { expect(result.status).toBe('failed'); expect(io.stderr()).toContain( - 'KTX Python runtime is required for this command. Run: ktx dev runtime install --feature local-embeddings --yes', + 'KTX Python runtime is required for this command. Run: ktx admin runtime install --feature local-embeddings --yes', ); }); @@ -331,7 +331,7 @@ describe('setup embeddings step', () => { expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:'); expect(config.ingest.embeddings.backend).toBe('none'); expect(io.stderr()).toContain('Local embedding health check failed: 401 invalid api key [redacted]'); - expect(io.stderr()).toContain('Prepare the runtime with: ktx dev runtime start --feature local-embeddings'); + expect(io.stderr()).toContain('Prepare the runtime with: ktx admin runtime start --feature local-embeddings'); expect(io.stderr()).not.toContain('skip for now'); }); diff --git a/packages/cli/src/setup-embeddings.ts b/packages/cli/src/setup-embeddings.ts index 475e5126..442b67bc 100644 --- a/packages/cli/src/setup-embeddings.ts +++ b/packages/cli/src/setup-embeddings.ts @@ -307,7 +307,7 @@ function localEmbeddingSetupMessage(message: string, stderrTail: string[] = []): const lines = [ `Local embedding health check failed: ${message}`, 'Local embeddings use the KTX-managed Python runtime.', - 'Prepare the runtime with: ktx dev runtime start --feature local-embeddings', + 'Prepare the runtime with: ktx admin runtime start --feature local-embeddings', 'Use --yes with setup to install and start the runtime without prompting.', 'The first run may download Python packages and the all-MiniLM-L6-v2 model.', ]; diff --git a/packages/cli/src/setup-runtime.test.ts b/packages/cli/src/setup-runtime.test.ts index ee070fc7..1d739423 100644 --- a/packages/cli/src/setup-runtime.test.ts +++ b/packages/cli/src/setup-runtime.test.ts @@ -71,7 +71,7 @@ describe('runKtxSetupRuntimeStep', () => { it('fails fast when required runtime features cannot be installed in no-input mode', async () => { const io = makeIo(); const ensureRuntime = vi.fn(async () => { - throw new Error('KTX Python runtime is required for this command. Run: ktx dev runtime install --yes'); + throw new Error('KTX Python runtime is required for this command. Run: ktx admin runtime install --yes'); }); await expect( @@ -94,7 +94,7 @@ describe('runKtxSetupRuntimeStep', () => { expect(ensureRuntime).toHaveBeenCalledWith(expect.objectContaining({ installPolicy: 'never' })); expect((await readKtxSetupState(tempDir)).completed_steps).not.toContain('runtime'); - expect(io.stderr()).toContain('ktx dev runtime install --yes'); + expect(io.stderr()).toContain('ktx admin runtime install --yes'); }); it('starts the managed local embeddings daemon for configured sentence-transformers embeddings', async () => { diff --git a/packages/cli/src/sl.ts b/packages/cli/src/sl.ts index d77fa4f9..4049936a 100644 --- a/packages/cli/src/sl.ts +++ b/packages/cli/src/sl.ts @@ -197,7 +197,7 @@ export async function runKtxSl(args: KtxSlArgs, io: KtxSlIo = process, deps: Ktx await printSlSources({ rows: sources, emptyMessage: `No semantic-layer sources matched "${args.query}" in ${project.projectDir}`, - emptyHint: 'Run `ktx sl list` to inspect available sources.', + emptyHint: 'Run `ktx sl` to inspect available sources.', command: 'sl search', output: args.output, json: args.json, diff --git a/packages/cli/src/standalone-smoke.test.ts b/packages/cli/src/standalone-smoke.test.ts index 688e69f2..3bfc74f2 100644 --- a/packages/cli/src/standalone-smoke.test.ts +++ b/packages/cli/src/standalone-smoke.test.ts @@ -144,6 +144,11 @@ describe('standalone built ktx CLI smoke', () => { expectSetupStderr(init); expect(init.stdout).toContain(`Project: ${projectDir}`); + const reindex = await runBuiltCli(['--project-dir', projectDir, 'admin', 'reindex', '--output', 'plain']); + expect(reindex.code).toBe(0); + expect(reindex.stdout).toContain('reindex\t'); + expect(reindex.stderr).toContain('wiki/global'); + const run = await runBuiltCli([ 'ingest', 'run', @@ -153,7 +158,7 @@ describe('standalone built ktx CLI smoke', () => { 'fake', ]); expect(run).toMatchObject({ code: 1, stdout: '' }); - expect(run.stderr).toContain("unknown option '--connection-id'"); + expect(run.stderr).toContain("unknown option '--adapter'"); }); it('rejects the removed agent command through the built binary', async () => { @@ -280,7 +285,7 @@ describe('standalone built ktx CLI smoke', () => { expect(add.code).toBe(1); expect(add.stdout).toBe(''); - expect(add.stderr).toContain("unknown command 'add'"); + expect(add.stderr).toMatch(/unknown (command|option)|too many arguments/); const yaml = await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'); expect(yaml).not.toContain('driver: notion'); diff --git a/packages/context/package.json b/packages/context/package.json index 104b4e47..11cfc28f 100644 --- a/packages/context/package.json +++ b/packages/context/package.json @@ -50,6 +50,11 @@ "import": "./dist/ingest/metabase-mapping.js", "default": "./dist/ingest/metabase-mapping.js" }, + "./index-sync": { + "types": "./dist/index-sync/index.d.ts", + "import": "./dist/index-sync/index.js", + "default": "./dist/index-sync/index.js" + }, "./scan": { "types": "./dist/scan/index.d.ts", "import": "./dist/scan/index.js", diff --git a/packages/context/src/index-sync/index.ts b/packages/context/src/index-sync/index.ts new file mode 100644 index 00000000..5863688e --- /dev/null +++ b/packages/context/src/index-sync/index.ts @@ -0,0 +1,2 @@ +export type { ReindexOptions, ReindexScopeResult, ReindexSummary, ReindexWorkResult } from './types.js'; +export { discoverReindexScopes, reindexLocalIndexes } from './reindex.js'; diff --git a/packages/context/src/index-sync/reindex.test.ts b/packages/context/src/index-sync/reindex.test.ts new file mode 100644 index 00000000..beb62342 --- /dev/null +++ b/packages/context/src/index-sync/reindex.test.ts @@ -0,0 +1,196 @@ +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import type { KtxEmbeddingPort } from '../core/index.js'; +import { initKtxProject, loadKtxProject, type KtxLocalProject } from '../project/index.js'; +import { SqliteKnowledgeIndex } from '../wiki/sqlite-knowledge-index.js'; +import { reindexLocalIndexes } from './reindex.js'; + +class FakeEmbeddingPort implements KtxEmbeddingPort { + readonly maxBatchSize = 8; + + async computeEmbedding(text: string): Promise { + return [text.length, 1]; + } + + async computeEmbeddingsBulk(texts: string[]): Promise { + return texts.map((text) => [text.length, 1]); + } +} + +async function createProject(tempDir: string): Promise { + await initKtxProject({ projectDir: tempDir, force: true }); + return loadKtxProject({ projectDir: tempDir }); +} + +describe('reindexLocalIndexes', () => { + let tempDir: string; + + beforeEach(async () => { + tempDir = await mkdtemp(join(tmpdir(), 'ktx-reindex-')); + }); + + afterEach(async () => { + await rm(tempDir, { recursive: true, force: true }); + }); + + it('returns an empty summary when no wiki or semantic-layer directories exist', async () => { + const project = await createProject(tempDir); + await rm(join(project.projectDir, 'wiki'), { recursive: true, force: true }); + await rm(join(project.projectDir, 'semantic-layer'), { recursive: true, force: true }); + + await expect(reindexLocalIndexes(project, { force: false, embeddingService: null })).resolves.toMatchObject({ + scopes: [], + totals: { scanned: 0, updated: 0, deleted: 0, embeddingsRecomputed: 0, embeddingsFailed: 0 }, + force: false, + embeddingsAvailable: false, + }); + }); + + it('discovers empty directories as zero-row scopes', async () => { + const project = await createProject(tempDir); + await mkdir(join(project.projectDir, 'wiki/user/local'), { recursive: true }); + await mkdir(join(project.projectDir, 'semantic-layer/warehouse'), { recursive: true }); + + const summary = await reindexLocalIndexes(project, { force: false, embeddingService: null }); + + expect(summary.scopes.map((scope) => scope.label)).toEqual(['global', 'user/local', 'warehouse']); + expect(summary.totals.scanned).toBe(0); + }); + + it('indexes mixed wiki and SL sources and reports totals', async () => { + const project = await createProject(tempDir); + await writeFile( + join(project.projectDir, 'wiki/global/revenue.md'), + '---\nsummary: Revenue\nusage_mode: auto\n---\n\nPaid orders.\n', + 'utf-8', + ); + await mkdir(join(project.projectDir, 'semantic-layer/warehouse'), { recursive: true }); + await writeFile( + join(project.projectDir, 'semantic-layer/warehouse/orders.yaml'), + 'name: orders\ntable: public.orders\ngrain: [id]\ncolumns:\n - name: id\n type: number\njoins: []\nmeasures: []\n', + 'utf-8', + ); + + const summary = await reindexLocalIndexes(project, { + force: false, + embeddingService: new FakeEmbeddingPort(), + }); + + expect(summary.scopes).toHaveLength(2); + expect(summary.totals).toMatchObject({ scanned: 2, updated: 2, deleted: 0, embeddingsRecomputed: 2 }); + expect(summary.embeddingsAvailable).toBe(true); + }); + + it('does not report unchanged lexical-only rows as updated on repeated runs', async () => { + const project = await createProject(tempDir); + await writeFile( + join(project.projectDir, 'wiki/global/revenue.md'), + '---\nsummary: Revenue\nusage_mode: auto\n---\n\nPaid orders.\n', + 'utf-8', + ); + await mkdir(join(project.projectDir, 'semantic-layer/warehouse'), { recursive: true }); + await writeFile( + join(project.projectDir, 'semantic-layer/warehouse/orders.yaml'), + 'name: orders\ntable: public.orders\ngrain: [id]\ncolumns:\n - name: id\n type: number\njoins: []\nmeasures: []\n', + 'utf-8', + ); + + const first = await reindexLocalIndexes(project, { force: false, embeddingService: null }); + expect(first.totals).toMatchObject({ + scanned: 2, + updated: 2, + deleted: 0, + embeddingsRecomputed: 0, + embeddingsFailed: 0, + }); + + const second = await reindexLocalIndexes(project, { force: false, embeddingService: null }); + + expect(second.totals).toMatchObject({ + scanned: 2, + updated: 0, + deleted: 0, + embeddingsRecomputed: 0, + embeddingsFailed: 0, + }); + expect(second.scopes.map((scope) => [scope.label, scope.updated])).toEqual([ + ['global', 0], + ['warehouse', 0], + ]); + }); + + it('force clears stale rows before rebuilding each discovered scope', async () => { + const project = await createProject(tempDir); + const wikiIndex = new SqliteKnowledgeIndex({ dbPath: join(project.projectDir, '.ktx/db.sqlite') }); + wikiIndex.sync([ + { + path: 'wiki/global/stale.md', + key: 'stale', + scope: 'GLOBAL', + scopeId: null, + summary: 'Stale', + content: 'Stale content', + tags: [], + embedding: [1, 0], + }, + ]); + await writeFile( + join(project.projectDir, 'wiki/global/revenue.md'), + '---\nsummary: Revenue\nusage_mode: auto\n---\n\nPaid orders.\n', + 'utf-8', + ); + + const summary = await reindexLocalIndexes(project, { + force: true, + embeddingService: new FakeEmbeddingPort(), + }); + + expect(summary.force).toBe(true); + expect(summary.totals).toMatchObject({ scanned: 1, updated: 1, deleted: 0 }); + expect(wikiIndex.search('Stale', 10)).toEqual([]); + }); + + it('captures a per-scope error and continues other scopes', async () => { + const project = await createProject(tempDir); + await writeFile( + join(project.projectDir, 'wiki/global/revenue.md'), + '---\nsummary: Revenue\nusage_mode: auto\n---\n\nPaid orders.\n', + 'utf-8', + ); + await mkdir(join(project.projectDir, 'semantic-layer/warehouse'), { recursive: true }); + await writeFile(join(project.projectDir, 'semantic-layer/warehouse/broken.yaml'), 'not: [valid', 'utf-8'); + + const summary = await reindexLocalIndexes(project, { force: false, embeddingService: null }); + + expect(summary.scopes.find((scope) => scope.label === 'global')?.error).toBeUndefined(); + expect(summary.scopes.find((scope) => scope.label === 'warehouse')?.error).toContain('YAML'); + }); + + it('marks a scope errored when configured embeddings fail', async () => { + const project = await createProject(tempDir); + await writeFile( + join(project.projectDir, 'wiki/global/revenue.md'), + '---\nsummary: Revenue\nusage_mode: auto\n---\n\nPaid orders.\n', + 'utf-8', + ); + const embeddingService: KtxEmbeddingPort = { + maxBatchSize: 8, + async computeEmbedding() { + throw new Error('embedding provider unavailable'); + }, + async computeEmbeddingsBulk() { + throw new Error('embedding provider unavailable'); + }, + }; + + const summary = await reindexLocalIndexes(project, { force: false, embeddingService }); + + expect(summary.scopes[0]).toMatchObject({ + label: 'global', + embeddingsFailed: 1, + error: '1 embedding recomputation failed', + }); + }); +}); diff --git a/packages/context/src/index-sync/reindex.ts b/packages/context/src/index-sync/reindex.ts new file mode 100644 index 00000000..d0cbe29a --- /dev/null +++ b/packages/context/src/index-sync/reindex.ts @@ -0,0 +1,162 @@ +import { readdir, stat } from 'node:fs/promises'; +import { join, relative } from 'node:path'; +import { ktxLocalStateDbPath, type KtxLocalProject } from '../project/index.js'; +import { loadLocalSlSourceRecords, SlSearchService, SqliteSlSourcesIndex } from '../sl/index.js'; +import { KnowledgeWikiService, SqliteKnowledgeIndex } from '../wiki/index.js'; +import type { ReindexOptions, ReindexScopeResult, ReindexSummary, ReindexWorkResult } from './types.js'; + +type DiscoveredScope = + | { kind: 'wiki'; scope: 'GLOBAL'; scopeId: null; label: 'global' } + | { kind: 'wiki'; scope: 'USER'; scopeId: string; label: `user/${string}` } + | { kind: 'sl'; connectionId: string; label: string }; + +const ZERO: ReindexWorkResult = { + scanned: 0, + updated: 0, + deleted: 0, + embeddingsRecomputed: 0, + embeddingsFailed: 0, +}; + +async function directoryExists(path: string): Promise { + try { + return (await stat(path)).isDirectory(); + } catch { + return false; + } +} + +async function childDirectories(path: string): Promise { + try { + const entries = await readdir(path, { withFileTypes: true }); + return entries + .filter((entry) => entry.isDirectory()) + .map((entry) => entry.name) + .sort((left, right) => left.localeCompare(right)); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + return []; + } + throw error; + } +} + +export async function discoverReindexScopes(project: KtxLocalProject): Promise { + const scopes: DiscoveredScope[] = []; + if (await directoryExists(join(project.projectDir, 'wiki/global'))) { + scopes.push({ kind: 'wiki', scope: 'GLOBAL', scopeId: null, label: 'global' }); + } + for (const userId of await childDirectories(join(project.projectDir, 'wiki/user'))) { + scopes.push({ kind: 'wiki', scope: 'USER', scopeId: userId, label: `user/${userId}` }); + } + for (const connectionId of await childDirectories(join(project.projectDir, 'semantic-layer'))) { + if (connectionId !== '_schema') { + scopes.push({ kind: 'sl', connectionId, label: connectionId }); + } + } + return scopes; +} + +function errorMessage(error: unknown): string { + if (!(error instanceof Error)) { + return String(error); + } + return error.name && error.name !== 'Error' ? `${error.name}: ${error.message}` : error.message; +} + +function addTotals(left: ReindexWorkResult, right: ReindexWorkResult): ReindexWorkResult { + return { + scanned: left.scanned + right.scanned, + updated: left.updated + right.updated, + deleted: left.deleted + right.deleted, + embeddingsRecomputed: left.embeddingsRecomputed + right.embeddingsRecomputed, + embeddingsFailed: left.embeddingsFailed + right.embeddingsFailed, + }; +} + +function durationSince(startedAt: bigint): number { + return Number((process.hrtime.bigint() - startedAt) / 1_000_000n); +} + +function embeddingFailureError(work: ReindexWorkResult): string | undefined { + if (work.embeddingsFailed === 0) { + return undefined; + } + return `${work.embeddingsFailed} embedding recomputation${work.embeddingsFailed === 1 ? '' : 's'} failed`; +} + +export async function reindexLocalIndexes( + project: KtxLocalProject, + options: ReindexOptions, +): Promise { + const startedAt = process.hrtime.bigint(); + const dbPath = ktxLocalStateDbPath(project); + const scopes = await discoverReindexScopes(project); + const wikiIndex = new SqliteKnowledgeIndex({ dbPath }); + const slIndex = new SqliteSlSourcesIndex({ dbPath }); + const wikiService = new KnowledgeWikiService(project.fileStore, options.embeddingService, wikiIndex, project.git); + const slService = new SlSearchService(options.embeddingService, slIndex); + const results: ReindexScopeResult[] = []; + + for (const scope of scopes) { + const scopeStartedAt = process.hrtime.bigint(); + try { + let work: ReindexWorkResult; + if (scope.kind === 'wiki') { + if (options.force) { + wikiIndex.clear(scope.scope, scope.scopeId); + } + work = await wikiService.syncIndex(scope.scope, scope.scopeId); + results.push({ + kind: 'wiki', + label: scope.label, + scope: scope.scope === 'GLOBAL' ? 'global' : 'user', + scopeId: scope.scopeId, + ...work, + ...(options.force ? { deleted: 0 } : {}), + ...(options.embeddingService && work.embeddingsFailed > 0 ? { error: embeddingFailureError(work) } : {}), + durationMs: durationSince(scopeStartedAt), + }); + continue; + } + + if (options.force) { + await slIndex.clear(scope.connectionId); + } + const records = await loadLocalSlSourceRecords(project, { connectionId: scope.connectionId }); + work = await slService.indexSources( + scope.connectionId, + records.map((record) => record.source), + ); + results.push({ + kind: 'sl', + label: scope.label, + connectionId: scope.connectionId, + ...work, + ...(options.force ? { deleted: 0 } : {}), + ...(options.embeddingService && work.embeddingsFailed > 0 ? { error: embeddingFailureError(work) } : {}), + durationMs: durationSince(scopeStartedAt), + }); + } catch (error) { + results.push({ + kind: scope.kind, + label: scope.label, + ...(scope.kind === 'wiki' + ? { scope: scope.scope === 'GLOBAL' ? 'global' : 'user', scopeId: scope.scopeId } + : { connectionId: scope.connectionId }), + ...ZERO, + durationMs: durationSince(scopeStartedAt), + error: errorMessage(error), + }); + } + } + + return { + scopes: results, + totals: results.reduce(addTotals, ZERO), + dbPath: relative(project.projectDir, dbPath) || dbPath, + force: options.force, + embeddingsAvailable: options.embeddingService !== null, + durationMs: durationSince(startedAt), + }; +} diff --git a/packages/context/src/index-sync/types.ts b/packages/context/src/index-sync/types.ts new file mode 100644 index 00000000..39970b57 --- /dev/null +++ b/packages/context/src/index-sync/types.ts @@ -0,0 +1,33 @@ +import type { KtxEmbeddingPort } from '../core/index.js'; + +export interface ReindexOptions { + force: boolean; + embeddingService: KtxEmbeddingPort | null; +} + +export interface ReindexWorkResult { + scanned: number; + updated: number; + deleted: number; + embeddingsRecomputed: number; + embeddingsFailed: number; +} + +export interface ReindexScopeResult extends ReindexWorkResult { + kind: 'wiki' | 'sl'; + label: string; + scope?: 'global' | 'user'; + scopeId?: string | null; + connectionId?: string; + durationMs: number; + error?: string; +} + +export interface ReindexSummary { + scopes: ReindexScopeResult[]; + totals: ReindexWorkResult; + dbPath: string; + force: boolean; + embeddingsAvailable: boolean; + durationMs: number; +} diff --git a/packages/context/src/index.ts b/packages/context/src/index.ts index dda42789..1441cad5 100644 --- a/packages/context/src/index.ts +++ b/packages/context/src/index.ts @@ -12,6 +12,7 @@ export * from './agent/index.js'; export * from './core/index.js'; export * from './daemon/index.js'; export * from './ingest/index.js'; +export * from './index-sync/index.js'; export * from './llm/index.js'; export type { CaptureSession, diff --git a/packages/context/src/ingest/local-bundle-runtime.ts b/packages/context/src/ingest/local-bundle-runtime.ts index bcb1258c..f5bb73bc 100644 --- a/packages/context/src/ingest/local-bundle-runtime.ts +++ b/packages/context/src/ingest/local-bundle-runtime.ts @@ -379,16 +379,19 @@ class LocalKnowledgeIndex implements KnowledgeIndexPort { return result; } - async deleteStale(): Promise { + async deleteStale(): Promise { await this.syncAllPagesFromDisk(); + return 0; } - async deleteByScope(): Promise { + async deleteByScope(): Promise { await this.syncAllPagesFromDisk(); + return 0; } - async deleteByKey(): Promise { + async deleteByKey(): Promise { await this.syncAllPagesFromDisk(); + return 0; } async findPageByKey(scope: string, scopeId: string | null, pageKey: string) { diff --git a/packages/context/src/memory/local-memory.ts b/packages/context/src/memory/local-memory.ts index a83c7a08..c12dec74 100644 --- a/packages/context/src/memory/local-memory.ts +++ b/packages/context/src/memory/local-memory.ts @@ -205,11 +205,17 @@ class LocalKnowledgeIndex implements KnowledgeIndexPort { return new Map(); } - async deleteStale(): Promise {} + async deleteStale(): Promise { + return 0; + } - async deleteByScope(): Promise {} + async deleteByScope(): Promise { + return 0; + } - async deleteByKey(): Promise {} + async deleteByKey(): Promise { + return 0; + } async findPageByKey(scope: string, scopeId: string | null, pageKey: string) { const path = this.pagePath(scope, scopeId, pageKey); diff --git a/packages/context/src/sl/ports.ts b/packages/context/src/sl/ports.ts index 888248d5..e01e1e73 100644 --- a/packages/context/src/sl/ports.ts +++ b/packages/context/src/sl/ports.ts @@ -40,9 +40,9 @@ export interface SlSourcesIndexPort { sources: Array<{ sourceName: string; searchText: string; embedding: number[] | null; contentHash?: string | null }>, ): Promise; getExistingSearchTexts(connectionId: string): Promise>; - deleteStale(connectionId: string, keepNames: string[]): Promise; - deleteByConnection(connectionId: string): Promise; - deleteByConnectionAndName(connectionId: string, sourceName: string): Promise; + deleteStale(connectionId: string, keepNames: string[]): Promise; + deleteByConnection(connectionId: string): Promise; + deleteByConnectionAndName(connectionId: string, sourceName: string): Promise; search( connectionId: string, queryEmbedding: number[] | null, diff --git a/packages/context/src/sl/sl-search.service.test.ts b/packages/context/src/sl/sl-search.service.test.ts index ffe27cbc..164c3954 100644 --- a/packages/context/src/sl/sl-search.service.test.ts +++ b/packages/context/src/sl/sl-search.service.test.ts @@ -223,4 +223,73 @@ describe('SlSearchService', () => { }, ]); }); + + it('indexSources reports stats and supports lexical-only indexing', async () => { + const repository = { + upsertSources: vi.fn().mockResolvedValue(undefined), + getExistingSearchTexts: vi.fn().mockResolvedValue( + new Map([ + ['old_source', { searchText: 'old source', hasEmbedding: true }], + ]), + ), + deleteStale: vi.fn().mockResolvedValue(1), + deleteByConnection: vi.fn().mockResolvedValue(0), + deleteByConnectionAndName: vi.fn(), + search: vi.fn(), + }; + const service = new SlSearchService(null, repository); + const source: SemanticLayerSource = { + name: 'orders', + table: 'public.orders', + grain: ['id'], + columns: [{ name: 'id', type: 'number' }], + joins: [], + measures: [], + }; + + await expect(service.indexSources('warehouse', [source])).resolves.toEqual({ + scanned: 1, + updated: 1, + deleted: 1, + embeddingsRecomputed: 0, + embeddingsFailed: 0, + }); + expect(repository.upsertSources).toHaveBeenCalledWith('warehouse', [ + expect.objectContaining({ sourceName: 'orders', embedding: null }), + ]); + }); + + it('does not update unchanged lexical-only SL rows on repeated sync', async () => { + const repository = { + upsertSources: vi.fn().mockResolvedValue(undefined), + getExistingSearchTexts: vi.fn().mockResolvedValue( + new Map([ + ['orders', { searchText: 'orders. table: public.orders. id (number)', hasEmbedding: false }], + ]), + ), + deleteStale: vi.fn().mockResolvedValue(0), + deleteByConnection: vi.fn().mockResolvedValue(0), + deleteByConnectionAndName: vi.fn(), + search: vi.fn(), + }; + const service = new SlSearchService(null, repository); + const source: SemanticLayerSource = { + name: 'orders', + table: 'public.orders', + grain: ['id'], + columns: [{ name: 'id', type: 'number' }], + joins: [], + measures: [], + }; + + await expect(service.indexSources('warehouse', [source])).resolves.toEqual({ + scanned: 1, + updated: 0, + deleted: 0, + embeddingsRecomputed: 0, + embeddingsFailed: 0, + }); + expect(repository.upsertSources).toHaveBeenCalledWith('warehouse', []); + expect(repository.deleteStale).toHaveBeenCalledWith('warehouse', ['orders']); + }); }); diff --git a/packages/context/src/sl/sl-search.service.ts b/packages/context/src/sl/sl-search.service.ts index 68ae1557..0a7ecfb5 100644 --- a/packages/context/src/sl/sl-search.service.ts +++ b/packages/context/src/sl/sl-search.service.ts @@ -1,5 +1,6 @@ import type { KtxEmbeddingPort, KtxLogger } from '../core/index.js'; import { noopLogger } from '../core/index.js'; +import type { ReindexWorkResult } from '../index-sync/types.js'; import { DEFAULT_PRIORITY, resolveDescription } from './descriptions.js'; import { normalizeSemanticLayerDescriptions } from './description-normalization.js'; import type { SlSourcesIndexPort } from './ports.js'; @@ -94,73 +95,71 @@ export function buildSemanticLayerSourceSearchText( export class SlSearchService { constructor( - private readonly embeddingService: KtxEmbeddingPort, + private readonly embeddingService: KtxEmbeddingPort | null, private readonly slSourcesRepository: SlSourcesIndexPort, private readonly logger: KtxLogger = noopLogger, ) {} - async indexSources(connectionId: string, sources: SemanticLayerSource[]): Promise { + async indexSources(connectionId: string, sources: SemanticLayerSource[]): Promise { + const existing = await this.slSourcesRepository.getExistingSearchTexts(connectionId); if (sources.length === 0) { - await this.slSourcesRepository.deleteByConnection(connectionId); - return; + const deleted = await this.slSourcesRepository.deleteByConnection(connectionId); + return { scanned: 0, updated: 0, deleted, embeddingsRecomputed: 0, embeddingsFailed: 0 }; } - // Detect which sources actually changed by comparing search_text - const existing = await this.slSourcesRepository.getExistingSearchTexts(connectionId); const searchTexts = sources.map((s) => this.buildSearchText(s)); + const embeddingService = this.embeddingService; const changedIndices: number[] = []; - for (let i = 0; i < sources.length; i++) { - const prev = existing.get(sources[i].name); - if (!prev || prev.searchText !== searchTexts[i] || !prev.hasEmbedding) { + for (let i = 0; i < sources.length; i += 1) { + const previous = existing.get(sources[i]!.name); + if ( + !previous || + previous.searchText !== searchTexts[i] || + (embeddingService !== null && !previous.hasEmbedding) + ) { changedIndices.push(i); } } - if (changedIndices.length === 0) { - // Still clean up stale sources even if nothing changed - const keepNames = sources.map((s) => s.name); - await this.slSourcesRepository.deleteStale(connectionId, keepNames); - this.logger.log(`SL sources for connection ${connectionId}: all ${sources.length} up to date, 0 reindexed`); - return; - } + let changedEmbeddings: (number[] | null)[] = changedIndices.map(() => null); + let embeddingsRecomputed = 0; + let embeddingsFailed = 0; - // Compute embeddings only for changed sources - const changedTexts = changedIndices.map((i) => searchTexts[i]); - let changedEmbeddings: (number[] | null)[]; - try { - const batchSize = this.embeddingService.maxBatchSize; - const allEmbeddings: number[][] = []; - for (let i = 0; i < changedTexts.length; i += batchSize) { - const batch = changedTexts.slice(i, i + batchSize); - const batchEmbeddings = await this.embeddingService.computeEmbeddingsBulk(batch); - allEmbeddings.push(...batchEmbeddings); + if (embeddingService && changedIndices.length > 0) { + try { + const changedTexts = changedIndices.map((index) => searchTexts[index]!); + const allEmbeddings: number[][] = []; + for (let i = 0; i < changedTexts.length; i += embeddingService.maxBatchSize) { + const batch = changedTexts.slice(i, i + embeddingService.maxBatchSize); + allEmbeddings.push(...(await embeddingService.computeEmbeddingsBulk(batch))); + } + changedEmbeddings = allEmbeddings; + embeddingsRecomputed = allEmbeddings.length; + } catch (error) { + this.logger.warn( + `Failed to compute SL source embeddings: ${error instanceof Error ? error.message : String(error)}`, + ); + embeddingsFailed = changedIndices.length; } - changedEmbeddings = allEmbeddings; - } catch (error) { - this.logger.warn( - `Failed to compute SL source embeddings: ${error instanceof Error ? error.message : String(error)}`, - ); - changedEmbeddings = changedIndices.map(() => null); } - const rows = changedIndices.map((srcIdx, i) => { - return { - sourceName: sources[srcIdx].name, - searchText: searchTexts[srcIdx], - embedding: changedEmbeddings[i], - }; - }); - + const rows = changedIndices.map((sourceIndex, embeddingIndex) => ({ + sourceName: sources[sourceIndex]!.name, + searchText: searchTexts[sourceIndex]!, + embedding: changedEmbeddings[embeddingIndex] ?? null, + })); await this.slSourcesRepository.upsertSources(connectionId, rows); - // Remove sources that no longer exist in YAML - const keepNames = sources.map((s) => s.name); - await this.slSourcesRepository.deleteStale(connectionId, keepNames); - - this.logger.log( - `SL sources for connection ${connectionId}: ${changedIndices.length}/${sources.length} reindexed, ${sources.length - changedIndices.length} unchanged`, - ); + const keepNames = sources.map((source) => source.name); + const deleted = await this.slSourcesRepository.deleteStale(connectionId, keepNames); + return { + scanned: sources.length, + updated: changedIndices.length, + deleted, + embeddingsRecomputed, + embeddingsFailed, + }; } async search( @@ -170,12 +169,14 @@ export class SlSearchService { minRrfScore = 0, ): Promise> { let queryEmbedding: number[] | null = null; - try { - queryEmbedding = await this.embeddingService.computeEmbedding(query); - } catch (error) { - this.logger.warn( - `Failed to compute query embedding, falling back to FTS + trigram: ${error instanceof Error ? error.message : String(error)}`, - ); + if (this.embeddingService) { + try { + queryEmbedding = await this.embeddingService.computeEmbedding(query); + } catch (error) { + this.logger.warn( + `Failed to compute query embedding, falling back to FTS + trigram: ${error instanceof Error ? error.message : String(error)}`, + ); + } } const results = await this.slSourcesRepository.search(connectionId, queryEmbedding, query, limit, minRrfScore); diff --git a/packages/context/src/sl/sqlite-sl-sources-index.test.ts b/packages/context/src/sl/sqlite-sl-sources-index.test.ts index 18258000..91a7727e 100644 --- a/packages/context/src/sl/sqlite-sl-sources-index.test.ts +++ b/packages/context/src/sl/sqlite-sl-sources-index.test.ts @@ -105,6 +105,33 @@ describe('SqliteSlSourcesIndex', () => { expect(await index.search('finance', null, 'revenue', 10)).toEqual([]); }); + it('clear removes sources and dictionary rows for one connection only', async () => { + const index = new SqliteSlSourcesIndex({ dbPath }); + await index.upsertSources('warehouse', [ + { sourceName: 'orders', searchText: 'orders revenue paid', embedding: null }, + ]); + await index.upsertSources('finance', [ + { sourceName: 'invoices', searchText: 'invoices revenue paid', embedding: null }, + ]); + await index.replaceDictionaryEntries('warehouse', [ + { connectionId: 'warehouse', sourceName: 'orders', columnName: 'status', value: 'paid', cardinality: 1 }, + ]); + await index.replaceDictionaryEntries('finance', [ + { connectionId: 'finance', sourceName: 'invoices', columnName: 'status', value: 'paid', cardinality: 1 }, + ]); + + await expect(index.clear('warehouse')).resolves.toBe(1); + + expect(await index.search('warehouse', null, 'revenue', 10)).toEqual([]); + expect(await index.search('finance', null, 'revenue', 10)).toEqual([ + expect.objectContaining({ sourceName: 'invoices' }), + ]); + await expect(index.searchDictionaryCandidates({ connectionIds: ['warehouse'], queryText: 'paid', limit: 10 })) + .resolves.toEqual([]); + await expect(index.searchDictionaryCandidates({ connectionIds: ['finance'], queryText: 'paid', limit: 10 })) + .resolves.toEqual([expect.objectContaining({ connectionId: 'finance', sourceName: 'invoices' })]); + }); + it('returns lane candidates with stable connection-scoped IDs', async () => { const index = new SqliteSlSourcesIndex({ dbPath }); diff --git a/packages/context/src/sl/sqlite-sl-sources-index.ts b/packages/context/src/sl/sqlite-sl-sources-index.ts index f53c8eef..cc078ffe 100644 --- a/packages/context/src/sl/sqlite-sl-sources-index.ts +++ b/packages/context/src/sl/sqlite-sl-sources-index.ts @@ -221,10 +221,9 @@ export class SqliteSlSourcesIndex implements SlSourcesIndexPort { ); } - async deleteStale(connectionId: string, keepNames: string[]): Promise { + async deleteStale(connectionId: string, keepNames: string[]): Promise { if (keepNames.length === 0) { - await this.deleteByConnection(connectionId); - return; + return this.deleteByConnection(connectionId); } const placeholders = keepNames.map(() => '?').join(', '); @@ -257,18 +256,29 @@ export class SqliteSlSourcesIndex implements SlSourcesIndexPort { }); remove(stale.map((row) => row.source_name)); + return stale.length; } - async deleteByConnection(connectionId: string): Promise { + async deleteByConnection(connectionId: string): Promise { + return this.clear(connectionId); + } + + async clear(connectionId: string): Promise { + const rows = this.db + .prepare('SELECT source_name FROM local_sl_sources WHERE connection_id = ?') + .all(connectionId) as Array<{ source_name: string }>; const remove = this.db.transaction(() => { this.db.prepare('DELETE FROM local_sl_sources_fts WHERE connection_id = ?').run(connectionId); this.db.prepare('DELETE FROM local_sl_sources WHERE connection_id = ?').run(connectionId); + this.db.prepare('DELETE FROM local_sl_dictionary_values_fts WHERE connection_id = ?').run(connectionId); + this.db.prepare('DELETE FROM local_sl_dictionary_values WHERE connection_id = ?').run(connectionId); }); remove(); + return rows.length; } - async deleteByConnectionAndName(connectionId: string, sourceName: string): Promise { - this.deleteByConnectionAndNameSync(connectionId, sourceName); + async deleteByConnectionAndName(connectionId: string, sourceName: string): Promise { + return this.deleteByConnectionAndNameSync(connectionId, sourceName); } async replaceDictionaryEntries(connectionId: string, entries: SlDictionaryEntry[]): Promise { @@ -537,7 +547,7 @@ export class SqliteSlSourcesIndex implements SlSourcesIndexPort { .filter((row) => row.rrfScore >= minRrfScore); } - private deleteByConnectionAndNameSync(connectionId: string, sourceName: string): void { + private deleteByConnectionAndNameSync(connectionId: string, sourceName: string): number { const remove = this.db.transaction(() => { this.db .prepare( @@ -548,7 +558,7 @@ export class SqliteSlSourcesIndex implements SlSourcesIndexPort { `, ) .run(connectionId, sourceName); - this.db + const result = this.db .prepare( ` DELETE FROM local_sl_sources @@ -557,7 +567,8 @@ export class SqliteSlSourcesIndex implements SlSourcesIndexPort { `, ) .run(connectionId, sourceName); + return Number(result.changes); }); - remove(); + return remove(); } } diff --git a/packages/context/src/wiki/knowledge-wiki.service.test.ts b/packages/context/src/wiki/knowledge-wiki.service.test.ts index f7bb86e4..d9242f97 100644 --- a/packages/context/src/wiki/knowledge-wiki.service.test.ts +++ b/packages/context/src/wiki/knowledge-wiki.service.test.ts @@ -4,9 +4,9 @@ import { KnowledgeWikiService, type WikiFrontmatter } from './knowledge-wiki.ser function makeService() { const pagesRepository: Record> = { upsertPage: vi.fn().mockResolvedValue(undefined), - deleteByKey: vi.fn().mockResolvedValue(undefined), - deleteByScope: vi.fn().mockResolvedValue(undefined), - deleteStale: vi.fn().mockResolvedValue(undefined), + deleteByKey: vi.fn().mockResolvedValue(0), + deleteByScope: vi.fn().mockResolvedValue(0), + deleteStale: vi.fn().mockResolvedValue(0), getExistingSearchTexts: vi.fn().mockResolvedValue(new Map()), applyDiffTransactional: vi.fn().mockResolvedValue(undefined), }; @@ -50,6 +50,87 @@ function makeService() { const fm: WikiFrontmatter = { summary: 'sum', usage_mode: 'auto' }; +describe('KnowledgeWikiService.syncIndex result stats', () => { + it('reports scanned, updated, deleted, and embedding counts', async () => { + const { service, pagesRepository, embeddingService, configService } = makeService(); + configService.listFiles.mockResolvedValue({ files: ['wiki/global/revenue.md'] }); + configService.readFile.mockResolvedValue({ + content: '---\nsummary: Revenue\nusage_mode: auto\ntags:\n - finance\n---\n\nPaid orders.\n', + }); + pagesRepository.getExistingSearchTexts.mockResolvedValue( + new Map([ + ['old-page', { searchText: 'old', hasEmbedding: true }], + ]), + ); + embeddingService.computeEmbeddingsBulk.mockResolvedValue([[0.1, 0.2, 0.3]]); + pagesRepository.deleteStale.mockResolvedValue(1); + + await expect(service.syncIndex('GLOBAL', null)).resolves.toEqual({ + scanned: 1, + updated: 1, + deleted: 1, + embeddingsRecomputed: 1, + embeddingsFailed: 0, + }); + }); + + it('indexes lexical rows when embeddings are not configured', async () => { + const { pagesRepository, configService, gitService, logger } = makeService(); + const service = new KnowledgeWikiService( + configService as any, + null, + pagesRepository as any, + gitService as any, + logger as any, + ); + configService.listFiles.mockResolvedValue({ files: ['wiki/global/revenue.md'] }); + configService.readFile.mockResolvedValue({ + content: '---\nsummary: Revenue\nusage_mode: auto\n---\n\nPaid orders.\n', + }); + pagesRepository.getExistingSearchTexts.mockResolvedValue(new Map()); + pagesRepository.deleteStale.mockResolvedValue(0); + + const result = await service.syncIndex('GLOBAL', null); + + expect(result.embeddingsRecomputed).toBe(0); + expect(result.embeddingsFailed).toBe(0); + expect(pagesRepository.upsertPage).toHaveBeenCalledWith( + expect.objectContaining({ pageKey: 'revenue', embedding: null }), + ); + }); + + it('does not update unchanged lexical-only wiki rows on repeated sync', async () => { + const { pagesRepository, configService, gitService, logger } = makeService(); + const service = new KnowledgeWikiService( + configService as any, + null, + pagesRepository as any, + gitService as any, + logger as any, + ); + configService.listFiles.mockResolvedValue({ files: ['wiki/global/revenue.md'] }); + configService.readFile.mockResolvedValue({ + content: '---\nsummary: Revenue\nusage_mode: auto\n---\n\nPaid orders.\n', + }); + pagesRepository.getExistingSearchTexts.mockResolvedValue( + new Map([ + ['revenue', { searchText: 'revenue\nRevenue\nPaid orders.', hasEmbedding: false }], + ]), + ); + pagesRepository.deleteStale.mockResolvedValue(0); + + await expect(service.syncIndex('GLOBAL', null)).resolves.toEqual({ + scanned: 1, + updated: 0, + deleted: 0, + embeddingsRecomputed: 0, + embeddingsFailed: 0, + }); + expect(pagesRepository.upsertPage).not.toHaveBeenCalled(); + expect(pagesRepository.deleteStale).toHaveBeenCalledWith('GLOBAL', null, ['revenue']); + }); +}); + describe('KnowledgeWikiService.forWorktree isolation', () => { it('syncSinglePage in worktree scope does not call pagesRepository.upsertPage', async () => { const { service, pagesRepository, embeddingService } = makeService(); diff --git a/packages/context/src/wiki/knowledge-wiki.service.ts b/packages/context/src/wiki/knowledge-wiki.service.ts index c8e276ab..88447c14 100644 --- a/packages/context/src/wiki/knowledge-wiki.service.ts +++ b/packages/context/src/wiki/knowledge-wiki.service.ts @@ -2,6 +2,7 @@ import { createHash } from 'node:crypto'; import YAML from 'yaml'; import type { KtxEmbeddingPort, KtxFileStorePort, KtxLogger } from '../core/index.js'; import { noopLogger } from '../core/index.js'; +import type { ReindexWorkResult } from '../index-sync/types.js'; import { assertFlatWikiKey, isFlatWikiKey } from './keys.js'; import { buildKnowledgeSearchText } from './knowledge-search-text.js'; import type { KnowledgeGitDiffPort, KnowledgeIndexPort, UpsertPageParams } from './ports.js'; @@ -16,7 +17,7 @@ export class KnowledgeWikiService { constructor( private readonly configService: KtxFileStorePort, - private readonly embeddingService: KtxEmbeddingPort, + private readonly embeddingService: KtxEmbeddingPort | null, private readonly pagesRepository: KnowledgeIndexPort, private readonly gitService: KnowledgeGitDiffPort, private readonly logger: KtxLogger = noopLogger, @@ -246,10 +247,12 @@ export class KnowledgeWikiService { const searchText = buildKnowledgeSearchText(pageKey, frontmatter.summary, content, frontmatter.tags); let embedding: number[] | null = null; - try { - embedding = await this.embeddingService.computeEmbedding(searchText); - } catch (err) { - this.logger.warn(`Embedding failed for page "${pageKey}": ${err instanceof Error ? err.message : String(err)}`); + if (this.embeddingService) { + try { + embedding = await this.embeddingService.computeEmbedding(searchText); + } catch (err) { + this.logger.warn(`Embedding failed for page "${pageKey}": ${err instanceof Error ? err.message : String(err)}`); + } } await this.pagesRepository.upsertPage({ @@ -269,14 +272,21 @@ export class KnowledgeWikiService { * Full sync: load all pages from disk for a scope, reindex changed pages, clean stale entries. * Mirrors SlSearchService.indexSources() pattern. */ - async syncIndex(scope: string, scopeId?: string | null): Promise { + async syncIndex(scope: string, scopeId?: string | null): Promise { const pageKeys = await this.listPageKeys(scope, scopeId); + const existing = await this.pagesRepository.getExistingSearchTexts(scope, scopeId ?? null); + if (pageKeys.length === 0) { - await this.pagesRepository.deleteByScope(scope, scopeId ?? null); - return; + const deleted = await this.pagesRepository.deleteByScope(scope, scopeId ?? null); + return { + scanned: 0, + updated: 0, + deleted, + embeddingsRecomputed: 0, + embeddingsFailed: 0, + }; } - // Load and parse all pages const pages: Array<{ pageKey: string; frontmatter: WikiFrontmatter; content: string; searchText: string }> = []; for (const key of pageKeys) { const page = await this.readPage(scope, scopeId, key); @@ -286,58 +296,58 @@ export class KnowledgeWikiService { } } - // Detect changes - const existing = await this.pagesRepository.getExistingSearchTexts(scope, scopeId ?? null); - const changedPages = pages.filter((p) => { - const ex = existing.get(p.pageKey); - return !ex || ex.searchText !== p.searchText || !ex.hasEmbedding; + const embeddingService = this.embeddingService; + const changedPages = pages.filter((page) => { + const previous = existing.get(page.pageKey); + return ( + !previous || + previous.searchText !== page.searchText || + (embeddingService !== null && !previous.hasEmbedding) + ); }); - if (changedPages.length === 0) { - // Still clean up stale - await this.pagesRepository.deleteStale(scope, scopeId ?? null, pageKeys); - this.logger.log(`Wiki sync ${scope}: all ${pages.length} pages up to date`); - return; - } + let embeddings: (number[] | null)[] = changedPages.map(() => null); + let embeddingsRecomputed = 0; + let embeddingsFailed = 0; - // Compute embeddings for changed pages (batched) - const changedTexts = changedPages.map((p) => p.searchText); - let embeddings: (number[] | null)[]; - try { - const batchSize = this.embeddingService.maxBatchSize; - const all: number[][] = []; - for (let i = 0; i < changedTexts.length; i += batchSize) { - const batch = changedTexts.slice(i, i + batchSize); - const batchEmb = await this.embeddingService.computeEmbeddingsBulk(batch); - all.push(...batchEmb); + if (embeddingService && changedPages.length > 0) { + try { + const changedTexts = changedPages.map((page) => page.searchText); + const all: number[][] = []; + for (let i = 0; i < changedTexts.length; i += embeddingService.maxBatchSize) { + const batch = changedTexts.slice(i, i + embeddingService.maxBatchSize); + all.push(...(await embeddingService.computeEmbeddingsBulk(batch))); + } + embeddings = all; + embeddingsRecomputed = all.length; + } catch (err) { + this.logger.warn(`Embedding batch failed during sync: ${err instanceof Error ? err.message : String(err)}`); + embeddingsFailed = changedPages.length; } - embeddings = all; - } catch (err) { - this.logger.warn(`Embedding batch failed during sync: ${err instanceof Error ? err.message : String(err)}`); - embeddings = changedPages.map(() => null); } - // Upsert changed pages - for (let i = 0; i < changedPages.length; i++) { - const p = changedPages[i]; + for (let i = 0; i < changedPages.length; i += 1) { + const page = changedPages[i]!; await this.pagesRepository.upsertPage({ scope, scopeId: scopeId ?? null, - pageKey: p.pageKey, - summary: p.frontmatter.summary, - usageMode: p.frontmatter.usage_mode, - sortOrder: p.frontmatter.sort_order ?? 0, - searchText: p.searchText, - embedding: embeddings[i], + pageKey: page.pageKey, + summary: page.frontmatter.summary, + usageMode: page.frontmatter.usage_mode, + sortOrder: page.frontmatter.sort_order ?? 0, + searchText: page.searchText, + embedding: embeddings[i] ?? null, }); } - // Clean stale entries - await this.pagesRepository.deleteStale(scope, scopeId ?? null, pageKeys); - - this.logger.log( - `Wiki sync ${scope}: ${changedPages.length}/${pages.length} reindexed, ${pages.length - changedPages.length} unchanged`, - ); + const deleted = await this.pagesRepository.deleteStale(scope, scopeId ?? null, pageKeys); + return { + scanned: pages.length, + updated: changedPages.length, + deleted, + embeddingsRecomputed, + embeddingsFailed, + }; } /** @@ -388,12 +398,14 @@ export class KnowledgeWikiService { parsed.frontmatter.tags, ); let embedding: number[] | null = null; - try { - embedding = await this.embeddingService.computeEmbedding(searchText); - } catch (err) { - this.logger.warn( - `[wiki.sync] embedding failed for ${parsedPath.pageKey}: ${err instanceof Error ? err.message : String(err)}`, - ); + if (this.embeddingService) { + try { + embedding = await this.embeddingService.computeEmbedding(searchText); + } catch (err) { + this.logger.warn( + `[wiki.sync] embedding failed for ${parsedPath.pageKey}: ${err instanceof Error ? err.message : String(err)}`, + ); + } } const contentHash = createHash('sha256').update(content).digest('hex'); upserts.push({ diff --git a/packages/context/src/wiki/ports.ts b/packages/context/src/wiki/ports.ts index 075a6c20..6bcb2f17 100644 --- a/packages/context/src/wiki/ports.ts +++ b/packages/context/src/wiki/ports.ts @@ -33,9 +33,9 @@ export interface KnowledgeIndexPort { scope: string, scopeId: string | null, ): Promise>; - deleteStale(scope: string, scopeId: string | null, keepKeys: string[]): Promise; - deleteByScope(scope: string, scopeId: string | null): Promise; - deleteByKey(scope: string, scopeId: string | null, pageKey: string): Promise; + deleteStale(scope: string, scopeId: string | null, keepKeys: string[]): Promise; + deleteByScope(scope: string, scopeId: string | null): Promise; + deleteByKey(scope: string, scopeId: string | null, pageKey: string): Promise; findPageByKey( scope: string, scopeId: string | null, diff --git a/packages/context/src/wiki/sqlite-knowledge-index.test.ts b/packages/context/src/wiki/sqlite-knowledge-index.test.ts index 2a45573d..940e9954 100644 --- a/packages/context/src/wiki/sqlite-knowledge-index.test.ts +++ b/packages/context/src/wiki/sqlite-knowledge-index.test.ts @@ -65,6 +65,35 @@ describe('SqliteKnowledgeIndex', () => { expect(index.search('churn', 10)).toEqual([]); }); + it('clear removes one wiki scope and leaves other scopes intact', async () => { + const index = new SqliteKnowledgeIndex({ dbPath }); + index.sync([ + page({ path: 'wiki/global/revenue.md', key: 'revenue', scope: 'GLOBAL', scopeId: null }), + page({ + path: 'wiki/user/local/revenue.md', + key: 'revenue', + scope: 'USER', + scopeId: 'local', + summary: 'Local revenue', + content: 'Local revenue notes.', + }), + page({ + path: 'wiki/user/alex/revenue.md', + key: 'revenue', + scope: 'USER', + scopeId: 'alex', + summary: 'Alex revenue', + content: 'Alex revenue notes.', + }), + ]); + + expect(index.clear('USER', 'local')).toBe(1); + + expect(index.search('Local', 10)).toEqual([]); + expect(index.search('Alex', 10)).toEqual([expect.objectContaining({ path: 'wiki/user/alex/revenue.md' })]); + expect(index.search('definition', 10)).toEqual([expect.objectContaining({ path: 'wiki/global/revenue.md' })]); + }); + it('exposes existing search text and embedding state for incremental refresh', () => { const index = new SqliteKnowledgeIndex({ dbPath }); index.sync([page({ path: 'wiki/global/revenue.md', key: 'revenue', embedding: [1, 0] })]); diff --git a/packages/context/src/wiki/sqlite-knowledge-index.ts b/packages/context/src/wiki/sqlite-knowledge-index.ts index 7a5ae8fc..66b30338 100644 --- a/packages/context/src/wiki/sqlite-knowledge-index.ts +++ b/packages/context/src/wiki/sqlite-knowledge-index.ts @@ -3,6 +3,7 @@ import { dirname } from 'node:path'; import Database from 'better-sqlite3'; import { buildKnowledgeSearchText } from './knowledge-search-text.js'; import type { LocalKnowledgeScope } from './local-knowledge.js'; +import type { KnowledgeIndexPageListing, UpsertPageParams } from './ports.js'; export interface SqliteKnowledgeIndexOptions { dbPath: string; @@ -12,6 +13,7 @@ export interface SqliteKnowledgeIndexPage { path: string; key: string; scope: LocalKnowledgeScope; + scopeId?: string | null; summary: string; content: string; tags: string[]; @@ -106,6 +108,7 @@ export class SqliteKnowledgeIndex { path TEXT PRIMARY KEY, key TEXT NOT NULL, scope TEXT NOT NULL, + scope_id TEXT, summary TEXT NOT NULL, content TEXT NOT NULL, tags TEXT NOT NULL, @@ -129,6 +132,9 @@ export class SqliteKnowledgeIndex { if (!columnNames.has('embedding_json')) { this.db.exec('ALTER TABLE knowledge_pages ADD COLUMN embedding_json TEXT'); } + if (!columnNames.has('scope_id')) { + this.db.exec('ALTER TABLE knowledge_pages ADD COLUMN scope_id TEXT'); + } } sync(pages: SqliteKnowledgeIndexPage[]): void { @@ -142,11 +148,12 @@ export class SqliteKnowledgeIndex { ? this.db.prepare('DELETE FROM knowledge_pages_fts') : this.db.prepare(`DELETE FROM knowledge_pages_fts WHERE path NOT IN (${keepPaths.map(() => '?').join(', ')})`); const upsertPage = this.db.prepare(` - INSERT INTO knowledge_pages (path, key, scope, summary, content, tags, search_text, embedding_json) - VALUES (@path, @key, @scope, @summary, @content, @tags, @searchText, @embeddingJson) + INSERT INTO knowledge_pages (path, key, scope, scope_id, summary, content, tags, search_text, embedding_json) + VALUES (@path, @key, @scope, @scopeId, @summary, @content, @tags, @searchText, @embeddingJson) ON CONFLICT(path) DO UPDATE SET key = excluded.key, scope = excluded.scope, + scope_id = excluded.scope_id, summary = excluded.summary, content = excluded.content, tags = excluded.tags, @@ -168,6 +175,7 @@ export class SqliteKnowledgeIndex { path: page.path, key: page.key, scope: page.scope, + scopeId: page.scopeId ?? null, summary: page.summary, content: searchText, tags: page.tags.join(' '), @@ -275,4 +283,201 @@ export class SqliteKnowledgeIndex { score: scoreFromRank(row.rawScore), })); } + + private pathForPage(scope: string, scopeId: string | null, pageKey: string): string { + return scope === 'GLOBAL' ? `wiki/global/${pageKey}.md` : `wiki/user/${scopeId ?? 'local'}/${pageKey}.md`; + } + + async upsertPage(params: UpsertPageParams): Promise { + const path = this.pathForPage(params.scope, params.scopeId, params.pageKey); + const row = { + path, + key: params.pageKey, + scope: params.scope, + scopeId: params.scopeId, + summary: params.summary, + content: params.searchText, + tags: '', + searchText: params.searchText, + embeddingJson: params.embedding && params.embedding.length > 0 ? JSON.stringify(params.embedding) : null, + }; + const write = this.db.transaction(() => { + this.db + .prepare( + ` + INSERT INTO knowledge_pages (path, key, scope, scope_id, summary, content, tags, search_text, embedding_json) + VALUES (@path, @key, @scope, @scopeId, @summary, @content, @tags, @searchText, @embeddingJson) + ON CONFLICT(path) DO UPDATE SET + key = excluded.key, + scope = excluded.scope, + scope_id = excluded.scope_id, + summary = excluded.summary, + content = excluded.content, + tags = excluded.tags, + search_text = excluded.search_text, + embedding_json = excluded.embedding_json + `, + ) + .run(row); + this.db.prepare('DELETE FROM knowledge_pages_fts WHERE path = @path').run(row); + this.db + .prepare( + ` + INSERT INTO knowledge_pages_fts (path, key, summary, content, tags) + VALUES (@path, @key, @summary, @content, @tags) + `, + ) + .run(row); + }); + write(); + } + + async getExistingSearchTexts( + scope: string, + scopeId: string | null, + ): Promise> { + const rows = this.db + .prepare( + ` + SELECT key, search_text, embedding_json + FROM knowledge_pages + WHERE scope = ? + AND scope_id IS ? + ORDER BY key ASC + `, + ) + .all(scope, scopeId) as Array<{ key: string; search_text: string; embedding_json: string | null }>; + return new Map( + rows.map((row) => [row.key, { searchText: row.search_text, hasEmbedding: row.embedding_json !== null }]), + ); + } + + async deleteStale(scope: string, scopeId: string | null, keepKeys: string[]): Promise { + if (keepKeys.length === 0) { + return this.deleteByScope(scope, scopeId); + } + const placeholders = keepKeys.map(() => '?').join(', '); + const stale = this.db + .prepare( + ` + SELECT key + FROM knowledge_pages + WHERE scope = ? + AND scope_id IS ? + AND key NOT IN (${placeholders}) + `, + ) + .all(scope, scopeId, ...keepKeys) as Array<{ key: string }>; + for (const row of stale) { + await this.deleteByKey(scope, scopeId, row.key); + } + return stale.length; + } + + async deleteByScope(scope: string, scopeId: string | null): Promise { + return this.clear(scope, scopeId); + } + + async deleteByKey(scope: string, scopeId: string | null, pageKey: string): Promise { + const path = this.pathForPage(scope, scopeId, pageKey); + const remove = this.db.transaction(() => { + this.db.prepare('DELETE FROM knowledge_pages_fts WHERE path = ?').run(path); + const result = this.db.prepare('DELETE FROM knowledge_pages WHERE path = ?').run(path); + return Number(result.changes); + }); + return remove(); + } + + clear(scope: string, scopeId: string | null): number { + const rows = this.db + .prepare('SELECT path FROM knowledge_pages WHERE scope = ? AND scope_id IS ?') + .all(scope, scopeId) as Array<{ path: string }>; + const remove = this.db.transaction((paths: string[]) => { + for (const path of paths) { + this.db.prepare('DELETE FROM knowledge_pages_fts WHERE path = ?').run(path); + this.db.prepare('DELETE FROM knowledge_pages WHERE path = ?').run(path); + } + }); + remove(rows.map((row) => row.path)); + return rows.length; + } + + async applyDiffTransactional(params: { + runId: string; + upserts: UpsertPageParams[]; + deletes: Array<{ scope: string; scopeId: string | null; pageKey: string }>; + }): Promise { + void params.runId; + for (const page of params.upserts) { + await this.upsertPage(page); + } + for (const page of params.deletes) { + await this.deleteByKey(page.scope, page.scopeId, page.pageKey); + } + } + + async findPageByKey( + scope: string, + scopeId: string | null, + pageKey: string, + ): Promise<{ id?: string; page_key: string } | null> { + const path = this.pathForPage(scope, scopeId, pageKey); + const row = this.db.prepare('SELECT path, key FROM knowledge_pages WHERE path = ?').get(path) as + | { path: string; key: string } + | undefined; + return row ? { id: row.path, page_key: row.key } : null; + } + + async listPagesForUser(userId: string): Promise { + const rows = this.db + .prepare( + ` + SELECT path, key, scope, scope_id, summary, tags + FROM knowledge_pages + WHERE scope = 'GLOBAL' + OR (scope = 'USER' AND scope_id = ?) + ORDER BY scope ASC, key ASC + `, + ) + .all(userId) as Array<{ + path: string; + key: string; + scope: string; + scope_id: string | null; + summary: string; + tags: string; + }>; + return rows.map((row) => ({ + id: row.path, + page_key: row.key, + summary: row.summary, + scope: row.scope, + scope_id: row.scope_id, + tags: row.tags.split(/\s+/).filter(Boolean), + })); + } + + async getUserPageCount(userId: string): Promise { + const row = this.db + .prepare("SELECT COUNT(*) AS count FROM knowledge_pages WHERE scope = 'USER' AND scope_id = ?") + .get(userId) as { count: number }; + return row.count; + } + + async incrementUsageCount(): Promise {} + + async searchRRF( + userId: string, + _embedding: number[] | null, + queryText: string, + limit: number, + ): Promise> { + const allowedPages = new Map((await this.listPagesForUser(userId)).map((page) => [page.id, page])); + return this.search(queryText, limit) + .map((row) => { + const page = allowedPages.get(row.path); + return page ? { pageKey: page.page_key, summary: page.summary, rrfScore: row.score } : null; + }) + .filter((row): row is { pageKey: string; summary: string; rrfScore: number } => row !== null); + } } diff --git a/scripts/check-boundaries.mjs b/scripts/check-boundaries.mjs index 3093fd42..1a2adf4f 100644 --- a/scripts/check-boundaries.mjs +++ b/scripts/check-boundaries.mjs @@ -11,6 +11,7 @@ const identifierAllowPatterns = [ /^packages\/cli\/src\/(?:index|managed-local-embeddings|managed-python-command|managed-python-daemon|managed-python-runtime|release-version|runtime)(?:\.test)?\.ts$/, /^python\/ktx-daemon\/src\/ktx_daemon\/__init__\.py$/, /^scripts\/(?:build-public-npm-package|build-python-runtime-wheel|local-embeddings-runtime-smoke|package-artifacts|public-npm-release-metadata|published-package-smoke|release-readiness)(?:\.test)?\.mjs$/, + /^scripts\/semantic-release-config\.cjs$/, ]; const forbiddenIdentifierTerms = ['kae' + 'lio', 'Kae' + 'lio', 'KAE' + 'LIO_']; diff --git a/scripts/check-boundaries.test.mjs b/scripts/check-boundaries.test.mjs index 832dc1c2..9fb8999d 100644 --- a/scripts/check-boundaries.test.mjs +++ b/scripts/check-boundaries.test.mjs @@ -79,6 +79,7 @@ describe('scanFileContent', () => { assert.equal(scanFileContent('scripts/local-embeddings-runtime-smoke.mjs', `@${name}/ktx`).length, 0); assert.equal(scanFileContent('scripts/package-artifacts.test.mjs', `${name}-ktx`).length, 0); assert.equal(scanFileContent('scripts/public-npm-release-metadata.mjs', `@${name}/ktx`).length, 0); + assert.equal(scanFileContent('scripts/semantic-release-config.cjs', `${name}-ktx-`).length, 0); assert.equal(scanFileContent('packages/cli/src/release-version.ts', `@${name}/ktx`).length, 0); assert.equal(scanFileContent('packages/cli/src/managed-python-runtime.ts', `${name}_ktx`).length, 0); assert.equal(scanFileContent('python/ktx-daemon/src/ktx_daemon/__init__.py', `${name}-ktx`).length, 0); diff --git a/scripts/conductor-setup.sh b/scripts/conductor-setup.sh index 8d64e7ad..81049e81 100755 --- a/scripts/conductor-setup.sh +++ b/scripts/conductor-setup.sh @@ -109,6 +109,8 @@ link_agent_overlays() { if [ -n "${CONDUCTOR_ROOT_PATH:-}" ]; then link_shared_path "$CONDUCTOR_ROOT_PATH/.agents" .agents || true link_shared_path "$CONDUCTOR_ROOT_PATH/.claude" .claude || true + mkdir -p docs + link_shared_path "$CONDUCTOR_ROOT_PATH/docs/superpowers" docs/superpowers || true fi } diff --git a/scripts/examples-docs.test.mjs b/scripts/examples-docs.test.mjs index bebe4965..a1f89847 100644 --- a/scripts/examples-docs.test.mjs +++ b/scripts/examples-docs.test.mjs @@ -165,10 +165,10 @@ describe('standalone example docs', () => { for (const command of [ 'ktx status --json', - 'ktx sl list --json', - 'ktx sl search "revenue" --json', + 'ktx sl --json', + 'ktx sl "revenue" --json', 'ktx sl query', - 'ktx wiki search "revenue recognition" --json', + 'ktx wiki "revenue recognition" --json', ]) { assert.match(servingAgents, new RegExp(command.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))); } @@ -193,12 +193,12 @@ describe('standalone example docs', () => { assert.match(rootReadme, publicPackagePattern('npm install -g {package}')); assert.match(quickstart, publicPackagePattern('npm install -g {package}')); - assert.match(quickstart, /ktx dev runtime install --feature local-embeddings --yes/); - assert.match(quickstart, /ktx dev runtime start --feature local-embeddings/); + assert.match(quickstart, /ktx admin runtime install --feature local-embeddings --yes/); + assert.match(quickstart, /ktx admin runtime start --feature local-embeddings/); assert.match(packageArtifacts, /requires `uv` on `PATH`/); - assert.match(packageArtifacts, /ktx dev runtime status/); - assert.match(packageArtifacts, /ktx dev runtime status/); - assert.doesNotMatch(packageArtifacts, /ktx dev runtime prune/); + assert.match(packageArtifacts, /ktx admin runtime status/); + assert.match(packageArtifacts, /ktx admin runtime status/); + assert.doesNotMatch(packageArtifacts, /ktx admin runtime prune/); assert.match( packageArtifacts, new RegExp( @@ -229,9 +229,9 @@ describe('standalone example docs', () => { assert.doesNotMatch(readme, /standalone Python distributions/); assert.doesNotMatch(readme, /installs the Python artifacts directly/); assert.match(readme, /requires `uv` on `PATH`/); - assert.match(readme, /ktx dev runtime status/); - assert.match(readme, /ktx dev runtime status/); - assert.doesNotMatch(readme, /ktx dev runtime prune/); + assert.match(readme, /ktx admin runtime status/); + assert.match(readme, /ktx admin runtime status/); + assert.doesNotMatch(readme, /ktx admin runtime prune/); assert.doesNotMatch(readme, /@ktx\/context/); assert.doesNotMatch(readme, /@ktx\/cli/); assert.doesNotMatch(readme, /python -m ktx_daemon semantic-validate/); @@ -241,7 +241,7 @@ describe('standalone example docs', () => { const rootReadme = await readText('README.md'); const cliMeta = await readText('docs-site/content/docs/cli-reference/meta.json'); const ingestReference = await readText('docs-site/content/docs/cli-reference/ktx-ingest.mdx'); - const devReference = await readText('docs-site/content/docs/cli-reference/ktx-dev.mdx'); + const adminReference = await readText('docs-site/content/docs/cli-reference/ktx-admin.mdx'); const setupReference = await readText('docs-site/content/docs/cli-reference/ktx-setup.mdx'); const buildingContext = await readText('docs-site/content/docs/guides/building-context.mdx'); const contextSources = await readText('docs-site/content/docs/integrations/context-sources.mdx'); @@ -252,7 +252,7 @@ describe('standalone example docs', () => { const localWarehouseReadme = await readText('examples/local-warehouse/README.md'); assert.match(ingestReference, /ktx ingest /); - assert.match(ingestReference, /ktx ingest --all --deep/); + assert.match(ingestReference, /Build every configured connection/); assert.match(ingestReference, /--query-history-window-days /); assert.match(buildingContext, /ktx ingest /); assert.match(buildingContext, /ktx ingest --all/); @@ -275,7 +275,7 @@ describe('standalone example docs', () => { assert.doesNotMatch(ingestReference, /--adapter/); assert.doesNotMatch(ingestReference, /ktx ingest watch/); assert.doesNotMatch(ingestReference, /live-database/); - assert.doesNotMatch(devReference, /ktx scan/); + assert.doesNotMatch(adminReference, /ktx scan/); assert.doesNotMatch(buildingContext, /ktx ingest watch/); assert.doesNotMatch(buildingContext, /ktx ingest status/); assert.doesNotMatch(buildingContext, /ktx ingest replay/); diff --git a/scripts/installed-live-database-smoke.mjs b/scripts/installed-live-database-smoke.mjs index 6337a582..3675016e 100644 --- a/scripts/installed-live-database-smoke.mjs +++ b/scripts/installed-live-database-smoke.mjs @@ -237,17 +237,17 @@ function parseDaemonBaseUrl(stdout) { } async function startDaemon(cleanInstallDir) { - const result = await run('pnpm', ['exec', 'ktx', 'dev', 'runtime', 'start'], { + const result = await run('pnpm', ['exec', 'ktx', 'admin', 'runtime', 'start'], { cwd: cleanInstallDir, env: managedRuntimeEnv(cleanInstallDir), timeout: 120_000, }); - requireSuccess('ktx dev runtime start', result); + requireSuccess('ktx admin runtime start', result); return parseDaemonBaseUrl(result.stdout); } async function stopDaemon(cleanInstallDir) { - await run('pnpm', ['exec', 'ktx', 'dev', 'runtime', 'stop'], { + await run('pnpm', ['exec', 'ktx', 'admin', 'runtime', 'stop'], { cwd: cleanInstallDir, env: managedRuntimeEnv(cleanInstallDir), timeout: 30_000, @@ -271,7 +271,7 @@ async function prepareCleanInstall(layout, cleanInstallDir) { await run('pnpm', ['install'], { cwd: cleanInstallDir, timeout: 120_000 }).then((result) => requireSuccess('pnpm install clean artifact project', result), ); - await run('pnpm', ['exec', 'ktx', 'dev', 'runtime', 'install', '--yes'], { + await run('pnpm', ['exec', 'ktx', 'admin', 'runtime', 'install', '--yes'], { cwd: cleanInstallDir, env: managedRuntimeEnv(cleanInstallDir), timeout: 120_000, diff --git a/scripts/local-embeddings-runtime-smoke.mjs b/scripts/local-embeddings-runtime-smoke.mjs index 652a039c..6c9c6ed0 100644 --- a/scripts/local-embeddings-runtime-smoke.mjs +++ b/scripts/local-embeddings-runtime-smoke.mjs @@ -74,27 +74,27 @@ export function localEmbeddingsSmokeCommands(input) { timeoutMs: 60_000, }, { - label: 'ktx dev runtime status missing', + label: 'ktx admin runtime status missing', command: 'pnpm', - args: ['exec', 'ktx', 'dev', 'runtime', 'status', '--json'], + args: ['exec', 'ktx', 'admin', 'runtime', 'status', '--json'], timeoutMs: 60_000, }, { - label: 'ktx dev runtime install local embeddings', + label: 'ktx admin runtime install local embeddings', command: 'pnpm', - args: ['exec', 'ktx', 'dev', 'runtime', 'install', '--feature', 'local-embeddings', '--yes'], + args: ['exec', 'ktx', 'admin', 'runtime', 'install', '--feature', 'local-embeddings', '--yes'], timeoutMs: 1_200_000, }, { - label: 'ktx dev runtime status local embeddings ready', + label: 'ktx admin runtime status local embeddings ready', command: 'pnpm', - args: ['exec', 'ktx', 'dev', 'runtime', 'status', '--json'], + args: ['exec', 'ktx', 'admin', 'runtime', 'status', '--json'], timeoutMs: 60_000, }, { - label: 'ktx dev runtime start local embeddings', + label: 'ktx admin runtime start local embeddings', command: 'pnpm', - args: ['exec', 'ktx', 'dev', 'runtime', 'start', '--feature', 'local-embeddings'], + args: ['exec', 'ktx', 'admin', 'runtime', 'start', '--feature', 'local-embeddings'], timeoutMs: 300_000, }, { @@ -118,9 +118,9 @@ export function localEmbeddingsSmokeCommands(input) { timeoutMs: 900_000, }, { - label: 'ktx dev runtime stop local embeddings', + label: 'ktx admin runtime stop local embeddings', command: 'pnpm', - args: ['exec', 'ktx', 'dev', 'runtime', 'stop'], + args: ['exec', 'ktx', 'admin', 'runtime', 'stop'], timeoutMs: 60_000, }, ]; @@ -374,7 +374,7 @@ export async function runLocalEmbeddingsRuntimeSmoke(options = {}) { process.stdout.write('KTX local embeddings runtime smoke verified\n'); } finally { if (daemonStarted) { - await run('pnpm', ['exec', 'ktx', 'dev', 'runtime', 'stop'], { + await run('pnpm', ['exec', 'ktx', 'admin', 'runtime', 'stop'], { cwd: installDir, env: smokeEnv, timeoutMs: 60_000, diff --git a/scripts/local-embeddings-runtime-smoke.test.mjs b/scripts/local-embeddings-runtime-smoke.test.mjs index f059f7a2..1b5cbfdd 100644 --- a/scripts/local-embeddings-runtime-smoke.test.mjs +++ b/scripts/local-embeddings-runtime-smoke.test.mjs @@ -89,23 +89,23 @@ describe('localEmbeddingsSmokeCommands', () => { assert.deepEqual(commands.map((command) => command.label), [ 'ktx public package version', - 'ktx dev runtime status missing', - 'ktx dev runtime install local embeddings', - 'ktx dev runtime status local embeddings ready', - 'ktx dev runtime start local embeddings', + 'ktx admin runtime status missing', + 'ktx admin runtime install local embeddings', + 'ktx admin runtime status local embeddings ready', + 'ktx admin runtime start local embeddings', 'ktx setup local embeddings', - 'ktx dev runtime stop local embeddings', + 'ktx admin runtime stop local embeddings', ]); assert.deepEqual(commands[2], { - label: 'ktx dev runtime install local embeddings', + label: 'ktx admin runtime install local embeddings', command: 'pnpm', - args: ['exec', 'ktx', 'dev', 'runtime', 'install', '--feature', 'local-embeddings', '--yes'], + args: ['exec', 'ktx', 'admin', 'runtime', 'install', '--feature', 'local-embeddings', '--yes'], timeoutMs: 1_200_000, }); assert.deepEqual(commands[4], { - label: 'ktx dev runtime start local embeddings', + label: 'ktx admin runtime start local embeddings', command: 'pnpm', - args: ['exec', 'ktx', 'dev', 'runtime', 'start', '--feature', 'local-embeddings'], + args: ['exec', 'ktx', 'admin', 'runtime', 'start', '--feature', 'local-embeddings'], timeoutMs: 300_000, }); assert.deepEqual(commands[5].args, [ diff --git a/scripts/package-artifacts.mjs b/scripts/package-artifacts.mjs index 3c885ac9..96a04bf0 100644 --- a/scripts/package-artifacts.mjs +++ b/scripts/package-artifacts.mjs @@ -147,7 +147,7 @@ export async function findPythonArtifacts(pythonDir) { files, RUNTIME_WHEEL_DISTRIBUTION_NAME, '.whl', - 'kaelio-ktx dev runtime wheel', + 'kaelio-ktx runtime wheel', pythonDir, RUNTIME_WHEEL_PACKAGE_VERSION, ), @@ -606,8 +606,8 @@ try { requireOutput('ktx public package version', version, await installedPackageVersionPattern()); const runtimeStatusBefore = parseJsonResultWithExitCode( - 'ktx dev runtime status missing', - await run('pnpm', ['exec', 'ktx', 'dev', 'runtime', 'status', '--json']), + 'ktx admin runtime status missing', + await run('pnpm', ['exec', 'ktx', 'admin', 'runtime', 'status', '--json']), 1, ); assert.equal(runtimeStatusBefore.kind, 'missing'); @@ -688,7 +688,6 @@ try { 'exec', 'ktx', 'wiki', - 'search', 'revenue', '--json', '--limit', @@ -731,7 +730,6 @@ try { 'exec', 'ktx', 'sl', - 'search', 'orders', '--json', '--connection-id', @@ -768,8 +766,8 @@ try { requireOutput('ktx sl query first managed runtime install', slQuery, /orders/); const runtimeStatusAfter = parseJsonResult( - 'ktx dev runtime status ready', - await run('pnpm', ['exec', 'ktx', 'dev', 'runtime', 'status', '--json']), + 'ktx admin runtime status ready', + await run('pnpm', ['exec', 'ktx', 'admin', 'runtime', 'status', '--json']), ); assert.equal(runtimeStatusAfter.kind, 'ready'); assert.deepEqual(runtimeStatusAfter.manifest.features, ['core']); @@ -797,29 +795,29 @@ try { requireOutput('ktx sl query sqlite execute', sqliteSlQuery, /"rows": \\[\\s*\\[\\s*3\\s*\\]\\s*\\]/); process.stdout.write('ktx sl query sqlite execute verified\\n'); - const runtimeDoctor = await run('pnpm', ['exec', 'ktx', 'dev', 'runtime', 'status']); - requireSuccess('ktx dev runtime status', runtimeDoctor); - requireOutput('ktx dev runtime status', runtimeDoctor, /KTX Python runtime/); - requireOutput('ktx dev runtime status', runtimeDoctor, /status: ready/); - process.stdout.write('ktx dev runtime status verified\\n'); + const runtimeDoctor = await run('pnpm', ['exec', 'ktx', 'admin', 'runtime', 'status']); + requireSuccess('ktx admin runtime status', runtimeDoctor); + requireOutput('ktx admin runtime status', runtimeDoctor, /KTX Python runtime/); + requireOutput('ktx admin runtime status', runtimeDoctor, /status: ready/); + process.stdout.write('ktx admin runtime status verified\\n'); - const runtimeStart = await run('pnpm', ['exec', 'ktx', 'dev', 'runtime', 'start']); - requireSuccess('ktx dev runtime start', runtimeStart); + const runtimeStart = await run('pnpm', ['exec', 'ktx', 'admin', 'runtime', 'start']); + requireSuccess('ktx admin runtime start', runtimeStart); daemonStarted = true; - requireOutput('ktx dev runtime start', runtimeStart, /Started KTX Python daemon/); - requireOutput('ktx dev runtime start', runtimeStart, /url: http:\\/\\/127\\.0\\.0\\.1:\\d+/); - requireOutput('ktx dev runtime start', runtimeStart, /features: core/); + requireOutput('ktx admin runtime start', runtimeStart, /Started KTX Python daemon/); + requireOutput('ktx admin runtime start', runtimeStart, /url: http:\\/\\/127\\.0\\.0\\.1:\\d+/); + requireOutput('ktx admin runtime start', runtimeStart, /features: core/); - const runtimeStartReuse = await run('pnpm', ['exec', 'ktx', 'dev', 'runtime', 'start']); - requireSuccess('ktx dev runtime start reuse', runtimeStartReuse); - requireOutput('ktx dev runtime start reuse', runtimeStartReuse, /Using existing KTX Python daemon/); - requireOutput('ktx dev runtime start reuse', runtimeStartReuse, /features: core/); + const runtimeStartReuse = await run('pnpm', ['exec', 'ktx', 'admin', 'runtime', 'start']); + requireSuccess('ktx admin runtime start reuse', runtimeStartReuse); + requireOutput('ktx admin runtime start reuse', runtimeStartReuse, /Using existing KTX Python daemon/); + requireOutput('ktx admin runtime start reuse', runtimeStartReuse, /features: core/); - const runtimeStop = await run('pnpm', ['exec', 'ktx', 'dev', 'runtime', 'stop']); - requireSuccess('ktx dev runtime stop', runtimeStop); + const runtimeStop = await run('pnpm', ['exec', 'ktx', 'admin', 'runtime', 'stop']); + requireSuccess('ktx admin runtime stop', runtimeStop); daemonStarted = false; - requireOutput('ktx dev runtime stop', runtimeStop, /Stopped KTX Python daemon/); - process.stdout.write('ktx dev runtime daemon lifecycle verified\\n'); + requireOutput('ktx admin runtime stop', runtimeStop, /Stopped KTX Python daemon/); + process.stdout.write('ktx admin runtime daemon lifecycle verified\\n'); const structuralScan = await run('pnpm', ['exec', 'ktx', 'ingest', 'warehouse', '--project-dir', @@ -849,7 +847,7 @@ try { process.stdout.write('ktx ingest state verified\\n'); } finally { if (daemonStarted) { - await run('pnpm', ['exec', 'ktx', 'dev', 'runtime', 'stop']); + await run('pnpm', ['exec', 'ktx', 'admin', 'runtime', 'stop']); } if (previousRuntimeRoot === undefined) { delete process.env.KTX_RUNTIME_ROOT; diff --git a/scripts/package-artifacts.test.mjs b/scripts/package-artifacts.test.mjs index 89d1a760..384eeb65 100644 --- a/scripts/package-artifacts.test.mjs +++ b/scripts/package-artifacts.test.mjs @@ -167,7 +167,7 @@ describe('findPythonArtifacts', () => { it('throws when a required Python artifact is missing', async () => { const root = await mkdtemp(join(tmpdir(), 'ktx-artifacts-test-')); try { - await assert.rejects(() => findPythonArtifacts(root), /Missing Python artifact: kaelio-ktx dev runtime wheel/); + await assert.rejects(() => findPythonArtifacts(root), /Missing Python artifact: kaelio-ktx runtime wheel/); } finally { await rm(root, { recursive: true, force: true }); } @@ -475,9 +475,9 @@ describe('verification snippets', () => { assert.doesNotMatch(source, /startSemanticDaemon/); assert.match(source, /run\('pnpm', \[\s*'exec',\s*'ktx',\s*'setup'/); assert.match(source, /wiki', 'global', 'revenue\.md'/); - assert.match(source, /run\('pnpm', \[\s*'exec',\s*'ktx',\s*'wiki',\s*'search'/); + assert.match(source, /run\('pnpm', \[\s*'exec',\s*'ktx',\s*'wiki',\s*'revenue'/); assert.match(source, /semantic-layer', 'warehouse', 'orders\.yaml'/); - assert.match(source, /run\('pnpm', \[\s*'exec',\s*'ktx',\s*'sl',\s*'search',\s*'orders'/); + assert.match(source, /run\('pnpm', \[\s*'exec',\s*'ktx',\s*'sl',\s*'orders'/); assert.match(source, /orders\.order_count/); assert.match(source, /node:sqlite/); assert.match(source, /driver: sqlite/); @@ -491,20 +491,20 @@ describe('verification snippets', () => { assert.doesNotMatch(source, /run\('python'/); assert.match(source, /KTX_RUNTIME_ROOT/); assert.match(source, /managed-runtime/); - assert.match(source, /ktx dev runtime status missing/); + assert.match(source, /ktx admin runtime status missing/); assert.match(source, /runtimeStatusBefore\.kind, 'missing'/); assert.ok(source.includes(String.raw`Installing KTX Python runtime \(core\) with uv`)); assert.match(source, /KTX Python runtime ready:/); - assert.match(source, /ktx dev runtime status ready/); + assert.match(source, /ktx admin runtime status ready/); assert.match(source, /runtimeStatusAfter\.kind, 'ready'/); assert.match(source, /runtimeStatusAfter\.manifest\.features/); - assert.match(source, /ktx dev runtime status/); + assert.match(source, /ktx admin runtime status/); assert.match(source, /status: ready/); - assert.match(source, /ktx dev runtime start/); - assert.match(source, /ktx dev runtime start reuse/); + assert.match(source, /ktx admin runtime start/); + assert.match(source, /ktx admin runtime start reuse/); assert.match(source, /Using existing KTX Python daemon/); - assert.match(source, /ktx dev runtime stop/); - assert.doesNotMatch(source, /ktx dev runtime prune/); + assert.match(source, /ktx admin runtime stop/); + assert.doesNotMatch(source, /ktx admin runtime prune/); assert.doesNotMatch(source, /staleRuntimeDir/); assert.match(source, /run\('pnpm', \[\s*'exec',\s*'ktx',\s*'ingest',\s*'warehouse'/); assert.match(source, /'--deep'/); diff --git a/scripts/semantic-release-config.cjs b/scripts/semantic-release-config.cjs index e9ec9cc5..fb1f959d 100644 --- a/scripts/semantic-release-config.cjs +++ b/scripts/semantic-release-config.cjs @@ -166,19 +166,10 @@ function createReleaseConfig(env = process.env) { 'pnpm run artifacts:check', 'pnpm run release:readiness', ].join(' && '), - }, - ], - [ - '@semantic-release/npm', - { - pkgRoot: 'dist/public-npm-package', - tarballDir: 'dist/artifacts/npm', - }, - ], - [ - '@semantic-release/exec', - { - publishCmd: 'pnpm run release:published-smoke', + publishCmd: [ + `npm publish dist/artifacts/npm/kaelio-ktx-\${nextRelease.version}.tgz --tag ${tag} --access public --provenance`, + 'pnpm run release:published-smoke', + ].join(' && '), }, ], ...releaseGitPlugins(kind), diff --git a/scripts/semantic-release-config.test.mjs b/scripts/semantic-release-config.test.mjs index 99cd53c4..c62f0fdf 100644 --- a/scripts/semantic-release-config.test.mjs +++ b/scripts/semantic-release-config.test.mjs @@ -27,20 +27,20 @@ describe('semantic-release config', () => { ]); const config = createReleaseConfig({ KTX_RELEASE_KIND: 'rc', GITHUB_REF_NAME: 'main' }); - assert.deepEqual( + assert.equal( config.plugins.find((plugin) => Array.isArray(plugin) && plugin[0] === '@semantic-release/npm'), - [ - '@semantic-release/npm', - { - pkgRoot: 'dist/public-npm-package', - tarballDir: 'dist/artifacts/npm', - }, - ], + undefined, + '@semantic-release/npm must not run; the exec publishCmd publishes the pre-built tarball', ); assert.match( releaseExecOptions(config).prepareCmd, /update-public-release-version\.mjs "\$\{nextRelease\.version\}" "next"/, ); + assert.match( + releaseExecOptions(config).publishCmd, + /^npm publish dist\/artifacts\/npm\/kaelio-ktx-\$\{nextRelease\.version\}\.tgz --tag next --access public --provenance/, + ); + assert.match(releaseExecOptions(config).publishCmd, /pnpm run release:published-smoke/); assert.doesNotMatch(JSON.stringify(config.plugins), /release:npm-publish/); const releaseFilePluginNames = pluginNames(config).filter( (plugin) => plugin === '@semantic-release/changelog' || plugin === '@semantic-release/git', @@ -62,6 +62,10 @@ describe('semantic-release config', () => { releaseExecOptions(config).prepareCmd, /update-public-release-version\.mjs "\$\{nextRelease\.version\}" "latest"/, ); + assert.match( + releaseExecOptions(config).publishCmd, + /^npm publish dist\/artifacts\/npm\/kaelio-ktx-\$\{nextRelease\.version\}\.tgz --tag latest --access public --provenance/, + ); assert.equal(config.plugins.includes('./scripts/semantic-release-version-policy.cjs'), false); });