mirror of
https://github.com/Kaelio/ktx.git
synced 2026-06-10 08:05:14 +02:00
Merge remote-tracking branch 'origin/main' into ktx-cli-ergonomics-review
# Conflicts: # docs-site/content/docs/cli-reference/ktx-ingest.mdx
This commit is contained in:
Andrey Avtomonov
commit
9190eaf306
59 changed files with 1691 additions and 449 deletions
|
|
@ -4,7 +4,6 @@ import { useCallback, useState } from "react";
|
|||
import {
|
||||
Background,
|
||||
BackgroundVariant,
|
||||
Controls,
|
||||
Handle,
|
||||
MarkerType,
|
||||
type Node,
|
||||
|
|
@ -1036,7 +1035,7 @@ export function SemanticLayerFlow() {
|
|||
}}
|
||||
>
|
||||
<div className="pointer-events-none absolute right-2.5 top-2.5 z-10 rounded border border-fd-border/50 bg-white/30 px-1.5 py-px font-mono text-[9.5px] font-medium uppercase tracking-[0.06em] text-fd-muted-foreground shadow-sm backdrop-blur-sm dark:bg-white/10">
|
||||
Pan / zoom
|
||||
Drag to pan • ⌘/Ctrl + scroll to zoom
|
||||
</div>
|
||||
<ReactFlow
|
||||
nodes={nodes}
|
||||
|
|
@ -1050,15 +1049,14 @@ export function SemanticLayerFlow() {
|
|||
elementsSelectable={false}
|
||||
panOnDrag
|
||||
panOnScroll={false}
|
||||
zoomOnScroll
|
||||
zoomOnScroll={false}
|
||||
zoomOnPinch
|
||||
zoomOnDoubleClick
|
||||
preventScrolling
|
||||
preventScrolling={false}
|
||||
minZoom={minZoom}
|
||||
maxZoom={1.5}
|
||||
proOptions={{ hideAttribution: true }}
|
||||
>
|
||||
<Controls position="bottom-right" showInteractive={false} />
|
||||
<Background
|
||||
variant={BackgroundVariant.Dots}
|
||||
gap={18}
|
||||
|
|
|
|||
|
|
@ -5,46 +5,46 @@ Set up KTX from scratch end-to-end as a fully autonomous, agent-driven replaceme
|
|||
# Operating principles
|
||||
|
||||
- **Be autonomous.** Detect, decide, and act. Only ask the user when you need information that only they can provide: project location, which databases/sources to connect, credentials, and similar choices.
|
||||
- **Stream short status updates.** Before each major phase ("Checking prerequisites…", "Installing uv…", "Configuring warehouse connection…", "Running fast ingest…") print a one-line update. Not chatty — just enough that the user can see what's happening.
|
||||
- **Stream short status updates.** Before each major phase ("Checking prerequisites…", "Installing uv…", "Configuring warehouse connection…", "Running fast ingest…") print a one-line update. Not chatty - just enough that the user can see what's happening.
|
||||
- **Verify against docs, never guess.** CLI flags, config keys, and command names must come from the docs or from `ktx <command> --help`. If something looks wrong or missing, say so explicitly.
|
||||
- **Print every command you run and its exit code.** Terse, not silent.
|
||||
- **Fail loudly with cause + fix.** When a command fails: capture the exact error, identify the cause, change something, retry. Never retry an unchanged command. Exceptions for *known soft-failures* are listed in Phase 4 — handle those without retrying.
|
||||
- **Fail loudly with cause + fix.** When a command fails: capture the exact error, identify the cause, change something, retry. Never retry an unchanged command. Exceptions for *known soft-failures* are listed in Phase 4 - handle those without retrying.
|
||||
- **No LLM-based ingestion in this flow.** Only `--fast` ingest (schema-only). The user can run `--deep` later.
|
||||
- **Platform-agnostic.** Detect the host OS first and pick the right install commands / path syntax. Anything path- or shell-specific must branch on OS.
|
||||
|
||||
# Authoritative docs
|
||||
|
||||
KTX docs are served at `https://docs.kaelio.com/ktx/`. **Start by fetching `https://docs.kaelio.com/ktx/llms.txt`** to discover the docs map. Scan it for a "troubleshooting" entry — if one exists, read it **before** running install/setup so you can apply known fixes preemptively rather than after failing. If no troubleshooting page is listed (current state of the docs), proceed. Then fetch any other `.md` pages you need (setup, ingest, status, connection types). **Never invent CLI flags or config keys** — verify against the docs or `ktx --help` / `ktx <subcommand> --help`.
|
||||
KTX docs are served at `https://docs.kaelio.com/ktx/`. **Start by fetching `https://docs.kaelio.com/ktx/llms.txt`** to discover the docs map. Scan it for a "troubleshooting" entry - if one exists, read it **before** running install/setup so you can apply known fixes preemptively rather than after failing. If no troubleshooting page is listed (current state of the docs), proceed. Then fetch any other `.md` pages you need (setup, ingest, status, connection types). **Never invent CLI flags or config keys** - verify against the docs or `ktx --help` / `ktx <subcommand> --help`.
|
||||
|
||||
> **Note on the `ktx status` JSON example in the docs.** The docs page for `ktx status` shows an example shaped like `{"title": "...", "checks": [...]}`. That example is outdated. The real CLI output uses a top-level `verdict` field plus a `connections[]` array — see Phase 5 for the canonical success criteria. Trust the shape in this prompt over the docs example.
|
||||
> **Note on the `ktx status` JSON example in the docs.** The docs page for `ktx status` shows an example shaped like `{"title": "...", "checks": [...]}`. That example is outdated. The real CLI output uses a top-level `verdict` field plus a `connections[]` array - see Phase 5 for the canonical success criteria. Trust the shape in this prompt over the docs example.
|
||||
|
||||
# Workflow
|
||||
|
||||
## Phase 1 — Detect environment
|
||||
## Phase 1 - Detect environment
|
||||
|
||||
Determine the host OS (e.g. via `uname -s`, `process.platform`, or `$env:OS`). Use the right install commands per OS for the rest of this flow.
|
||||
|
||||
| Tool | macOS / Linux | Windows (PowerShell) |
|
||||
|------|---------------|----------------------|
|
||||
| `uv` | `curl -LsSf https://astral.sh/uv/install.sh \| sh` then re-source shell env | `irm https://astral.sh/uv/install.ps1 \| iex` |
|
||||
| Node.js | use system / fnm / nvm — **do not** auto-install | use system / nvm-windows — **do not** auto-install |
|
||||
| Node.js | use system / fnm / nvm - **do not** auto-install | use system / nvm-windows - **do not** auto-install |
|
||||
| KTX CLI | `npm install -g …` (see Phase 2) | `npm install -g …` (see Phase 2) |
|
||||
|
||||
If Node.js is missing, **stop and ask the user** to install it (https://nodejs.org/). Do not attempt to auto-install Node.
|
||||
|
||||
## Phase 2 — Verify and install prerequisites
|
||||
## Phase 2 - Verify and install prerequisites
|
||||
|
||||
Check each tool in order; install only if missing.
|
||||
|
||||
1. **Node.js** — run `node --version`. Require >= 22. If missing or older, stop and instruct the user.
|
||||
2. **`uv`** — run `uv --version`. If missing, run the OS-appropriate install command, then re-source the shell environment (`export PATH="$HOME/.local/bin:$PATH"` on Linux/macOS) so `uv` is on `PATH`.
|
||||
3. **KTX CLI** —
|
||||
1. **Node.js** - run `node --version`. Require >= 22. If missing or older, stop and instruct the user.
|
||||
2. **`uv`** - run `uv --version`. If missing, run the OS-appropriate install command, then re-source the shell environment (`export PATH="$HOME/.local/bin:$PATH"` on Linux/macOS) so `uv` is on `PATH`.
|
||||
3. **KTX CLI** -
|
||||
- Install ktx with `npm install -g @kaelio/ktx`
|
||||
- Verify with `ktx --version`.
|
||||
|
||||
Print one status line per tool ("✓ uv 0.11.15 found", "Installing uv…", "✓ ktx 0.x.y installed").
|
||||
|
||||
## Phase 3 — Gather user choices
|
||||
## Phase 3 - Gather user choices
|
||||
|
||||
Ask the user (grouped if your harness supports it; otherwise sequentially):
|
||||
|
||||
|
|
@ -55,14 +55,14 @@ Ask the user (grouped if your harness supports it; otherwise sequentially):
|
|||
- Connection name (e.g. `warehouse`, `analytics`).
|
||||
- Driver: one of `sqlite`, `postgres`, `mysql`, `sqlserver`, `bigquery`, `snowflake`.
|
||||
- Connection URL/DSN (or service-account file for BigQuery). Accept `env:VAR_NAME` or `file:/abs/path` to avoid pasting raw secrets.
|
||||
- **Heads-up for the user**: even if they paste a literal URL, KTX will silently relocate it into `<project>/.ktx/secrets/<connection>-url` and rewrite `ktx.yaml` to `url: file:…` — this is correct, secure behavior and not a bug.
|
||||
- **Heads-up for the user**: even if they paste a literal URL, KTX will silently relocate it into `<project>/.ktx/secrets/<connection>-url` and rewrite `ktx.yaml` to `url: file:…` - this is correct, secure behavior and not a bug.
|
||||
- Schemas / datasets to include (postgres / sqlserver / snowflake / bigquery only).
|
||||
- Optional `enabled_tables` allowlist if the user wants to scope ingest to specific tables.
|
||||
5. **BI / metadata sources** (dbt, Metabase, Looker, LookML, MetricFlow, Notion). Default: none. Ask only if the user mentions them.
|
||||
|
||||
## Phase 4 — Configure the project
|
||||
## Phase 4 - Configure the project
|
||||
|
||||
Drive the existing wizard non-interactively (verify exact flag names with `ktx setup --help` and the docs — the automation flags are hidden from help but accepted):
|
||||
Drive the existing wizard non-interactively (verify exact flag names with `ktx setup --help` and the docs - the automation flags are hidden from help but accepted):
|
||||
|
||||
```
|
||||
ktx setup \
|
||||
|
|
@ -107,17 +107,17 @@ This is **expected** and **does not mean setup failed**. Treat the exit code as
|
|||
- `ktx connection test <name>` (run next) exits 0 for every connection.
|
||||
- `ktx status --json --no-input` reports `verdict: "ready"`.
|
||||
|
||||
If those three conditions hold, proceed to Phase 5 without retrying setup, and **do not** switch to `--deep` to "fix" the readiness gate — deep ingest is explicitly out of scope. Mention this in the final report under "Docs / CLI gaps" so the user is aware.
|
||||
If those three conditions hold, proceed to Phase 5 without retrying setup, and **do not** switch to `--deep` to "fix" the readiness gate - deep ingest is explicitly out of scope. Mention this in the final report under "Docs / CLI gaps" so the user is aware.
|
||||
|
||||
If any of those three conditions do not hold, this is a real failure — capture the error, fetch the relevant docs page, fix the cause, retry.
|
||||
If any of those three conditions do not hold, this is a real failure - capture the error, fetch the relevant docs page, fix the cause, retry.
|
||||
|
||||
After `ktx setup` writes `ktx.yaml`, edit it directly for anything flags don't cover:
|
||||
- Per-connection `enabled_tables` allowlist (snake_case, under `connections.<name>.enabled_tables`).
|
||||
- Any advanced settings the user requested.
|
||||
|
||||
Use a YAML-aware editor (e.g. `uv run python -c "import yaml; …"`) — do not hand-edit blindly.
|
||||
Use a YAML-aware editor (e.g. `uv run python -c "import yaml; …"`) - do not hand-edit blindly.
|
||||
|
||||
## Phase 5 — Verify
|
||||
## Phase 5 - Verify
|
||||
|
||||
`ktx setup` already runs a fast schema ingest of every database connection it configures, so you do not need to re-ingest by default. For each configured connection:
|
||||
|
||||
|
|
@ -139,25 +139,25 @@ Then run the global health check:
|
|||
ktx status --json --no-input
|
||||
```
|
||||
|
||||
Success requires (canonical shape — supersedes the example in the docs):
|
||||
Success requires (canonical shape - supersedes the example in the docs):
|
||||
- `verdict: "ready"` at the top of the JSON.
|
||||
- Every `connections[].status === "ok"`.
|
||||
- `ktx connection test <name>` exited 0 for every connection.
|
||||
|
||||
Do **not** run `--deep` ingest in this flow — that requires LLM time and is out of scope.
|
||||
Do **not** run `--deep` ingest in this flow - that requires LLM time and is out of scope.
|
||||
|
||||
### Optional: directly probe the embeddings daemon
|
||||
|
||||
If the user asks for stronger verification that `sentence-transformers` is actually serving (not just that setup said "ok"), do all of:
|
||||
|
||||
1. `ktx dev runtime status --json` → expect `"kind": "ready"` and `"features": [..., "local-embeddings"]`.
|
||||
1. `ktx admin runtime status --json` → expect `"kind": "ready"` and `"features": [..., "local-embeddings"]`.
|
||||
2. `pgrep -fa ktx-daemon` → expect a process running `ktx-daemon serve-http`.
|
||||
3. `curl -sS http://127.0.0.1:<port>/health` → expect HTTP 200 with `{"status":"healthy",…}`.
|
||||
4. `curl -sS -X POST http://127.0.0.1:<port>/embeddings/compute -H 'content-type: application/json' -d '{"text":"hello"}'` → expect `{"embedding": [...384 floats...]}`.
|
||||
|
||||
Discover the port from setup's log line `Started KTX local embeddings daemon: http://127.0.0.1:<port>` or from the daemon's OpenAPI at `GET /openapi.json`. Note: the routes are `/health` and `/embeddings/compute` — not `/healthz` or `/embeddings`.
|
||||
Discover the port from setup's log line `Started KTX local embeddings daemon: http://127.0.0.1:<port>` or from the daemon's OpenAPI at `GET /openapi.json`. Note: the routes are `/health` and `/embeddings/compute` - not `/healthz` or `/embeddings`.
|
||||
|
||||
## Phase 6 — Final report
|
||||
## Phase 6 - Final report
|
||||
|
||||
Print a structured report:
|
||||
|
||||
|
|
@ -180,7 +180,7 @@ Verdict: ready
|
|||
Then **Next steps** (copy-pasteable):
|
||||
1. Enrich with AI descriptions and embeddings: `ktx ingest <connection> --deep` (several minutes per connection).
|
||||
2. Add more connections later by rerunning this setup or via `ktx setup --database … --database-connection-id …`.
|
||||
3. Configure BI sources (dbt, Metabase, Looker, LookML, MetricFlow, Notion) — see `ktx setup --help` for `--source …` flags.
|
||||
3. Configure BI sources (dbt, Metabase, Looker, LookML, MetricFlow, Notion) - see `ktx setup --help` for `--source …` flags.
|
||||
4. Install agent integration: `ktx setup --agents --target <claude-code|claude-desktop|codex|cursor|opencode|universal>` (with optional `--global` for `claude-code`/`codex`).
|
||||
5. Connect the agent / MCP: see docs at `https://docs.kaelio.com/ktx/`.
|
||||
|
||||
|
|
@ -190,12 +190,12 @@ Under **Docs / CLI gaps to flag** include any of these that applied during your
|
|||
- `ktx status --json` real shape (`verdict`, `connections[]`) doesn't match the example in the docs page.
|
||||
- The pasted DB URL was moved to `.ktx/secrets/<name>-url` automatically.
|
||||
|
||||
End with a single line: `RESULT: PASS` or `RESULT: FAIL — <one-line reason>`.
|
||||
End with a single line: `RESULT: PASS` or `RESULT: FAIL - <one-line reason>`.
|
||||
|
||||
# Operating rules (recap)
|
||||
|
||||
- Print every command you run and its exit code. Status updates may be terse, but never silent.
|
||||
- On failure: capture the error, fetch the relevant docs page, fix the cause, retry. Never retry an unchanged command.
|
||||
- Known soft-failures (listed in Phase 4 and Phase 5) are not real failures — handle them as documented; do not retry or escalate.
|
||||
- Known soft-failures (listed in Phase 4 and Phase 5) are not real failures - handle them as documented; do not retry or escalate.
|
||||
- If you find a docs/CLI gap ("docs say X but CLI does Y"), call it out in the final report.
|
||||
- Never commit credentials — KTX accepts `env:` and `file:` references; prefer those. KTX will also auto-relocate literal URLs into `.ktx/secrets/`, but that does not protect anyone who pasted the URL into chat history.
|
||||
- Never commit credentials - KTX accepts `env:` and `file:` references; prefer those. KTX will also auto-relocate literal URLs into `.ktx/secrets/`, but that does not protect anyone who pasted the URL into chat history.
|
||||
|
|
|
|||
121
docs-site/content/docs/cli-reference/ktx-admin.mdx
Normal file
121
docs-site/content/docs/cli-reference/ktx-admin.mdx
Normal file
|
|
@ -0,0 +1,121 @@
|
|||
---
|
||||
title: "ktx admin"
|
||||
description: "Low-level project initialization, runtime, and index management."
|
||||
---
|
||||
|
||||
`ktx admin` contains low-level project initialization, managed Python runtime,
|
||||
and local index management commands. Context building lives at the root as
|
||||
[`ktx ingest`](/docs/cli-reference/ktx-ingest). Most users should start with
|
||||
`ktx setup`; use `ktx admin` when preparing local fixtures, checking the bundled
|
||||
runtime, rebuilding local indexes, or debugging runtime state.
|
||||
|
||||
## Command signature
|
||||
|
||||
```bash
|
||||
ktx admin <subcommand> [options]
|
||||
```
|
||||
|
||||
## Subcommands
|
||||
|
||||
| Subcommand | Description |
|
||||
|-----------|-------------|
|
||||
| `init [directory]` | Initialize a Git-backed KTX project directory for maintenance scripts |
|
||||
| `schema` | Print a JSON Schema describing `ktx.yaml` |
|
||||
| `runtime` | Install, start, stop, and inspect the KTX-managed Python runtime |
|
||||
| `reindex` | Sync local wiki and semantic-layer search indexes from disk |
|
||||
|
||||
## `admin init`
|
||||
|
||||
| Flag | Description | Default |
|
||||
|------|-------------|---------|
|
||||
| `--force` | Rewrite `ktx.yaml` and scaffold files in an existing project | `false` |
|
||||
|
||||
## `admin schema`
|
||||
|
||||
`ktx admin schema` does not require a `ktx.yaml` file or a configured project
|
||||
directory. Use it from any directory to generate editor or agent schema files.
|
||||
|
||||
| Flag | Description | Default |
|
||||
|------|-------------|---------|
|
||||
| `--output <file>` | Write the schema to a file instead of stdout | - |
|
||||
|
||||
## `admin runtime` Subcommands
|
||||
|
||||
| Subcommand | Description |
|
||||
|-----------|-------------|
|
||||
| `install` | Install the bundled Python runtime wheel into the managed runtime |
|
||||
| `start` | Start the KTX-managed Python HTTP daemon |
|
||||
| `stop` | Stop the KTX-managed Python HTTP daemon |
|
||||
| `status` | Show managed Python runtime status and readiness checks |
|
||||
|
||||
## `admin runtime` Options
|
||||
|
||||
| Flag | Description | Default |
|
||||
|------|-------------|---------|
|
||||
| `--feature <feature>` | Runtime feature level for `install` and `start` (`core` or `local-embeddings`) | `core` |
|
||||
| `--json` | Print JSON output for `status` | `false` |
|
||||
| `--yes` | Accepted by `install` for scripted install commands | `false` |
|
||||
| `--force` | Reinstall for `install`, or restart for `start` | `false` |
|
||||
| `--all` | Stop all recorded or discoverable KTX daemon processes with `stop` | `false` |
|
||||
|
||||
## Examples
|
||||
|
||||
```bash
|
||||
ktx admin init
|
||||
ktx admin init ./my-project
|
||||
ktx admin init --force
|
||||
|
||||
ktx admin schema
|
||||
ktx admin schema --output ./ktx.schema.json
|
||||
|
||||
ktx admin runtime install --yes
|
||||
ktx admin runtime install --feature local-embeddings --yes
|
||||
ktx admin runtime status
|
||||
ktx admin runtime start
|
||||
ktx admin runtime start --feature local-embeddings
|
||||
ktx admin runtime stop
|
||||
ktx admin runtime stop --all
|
||||
|
||||
ktx admin reindex
|
||||
ktx admin reindex --force
|
||||
ktx admin reindex --output plain
|
||||
ktx admin reindex --json
|
||||
```
|
||||
|
||||
## Output
|
||||
|
||||
Runtime commands print the runtime root, installed features, daemon URL, daemon
|
||||
pid, and log paths where relevant. `ktx admin runtime status --json` includes the
|
||||
runtime status plus readiness checks.
|
||||
|
||||
## `admin reindex`
|
||||
|
||||
`ktx admin reindex` syncs local wiki and semantic-layer search indexes from
|
||||
files on disk into `.ktx/db.sqlite`. The command discovers `wiki/global/`, each
|
||||
`wiki/user/<userId>/` directory, and each `semantic-layer/<connectionId>/`
|
||||
directory except `_schema`.
|
||||
|
||||
```bash
|
||||
ktx admin reindex
|
||||
ktx admin reindex --force
|
||||
ktx admin reindex --output plain
|
||||
ktx admin reindex --json
|
||||
```
|
||||
|
||||
By default, KTX compares stored search text with the files on disk. It only
|
||||
re-embeds changed rows and removes rows for files that no longer exist. With
|
||||
`--force`, KTX clears each discovered scope first and then rebuilds it.
|
||||
|
||||
When embeddings are not configured, KTX still writes lexical FTS rows and
|
||||
prints an embeddings warning. If a scope fails, KTX keeps processing the
|
||||
remaining scopes and exits with code `1` after output is written. If the local
|
||||
state database cannot open or the configured managed embedding runtime is
|
||||
missing, KTX prints the error and exits with code `1`.
|
||||
|
||||
## Common errors
|
||||
|
||||
| Error | Cause | Recovery |
|
||||
|-------|-------|----------|
|
||||
| Runtime status reports missing pieces | Packages, Python environment, or linked CLI are not ready | Run `pnpm install`, `pnpm run setup:dev`, `uv sync --all-groups`, then `ktx admin runtime status` |
|
||||
| Runtime daemon does not start | The managed Python runtime is missing or stale | Run `ktx admin runtime install --yes`, then `ktx admin runtime start` |
|
||||
| Multiple daemon processes remain | Older daemon state files or stray processes exist | Run `ktx admin runtime stop --all`, then start the runtime again |
|
||||
|
|
@ -1,91 +0,0 @@
|
|||
---
|
||||
title: "ktx dev"
|
||||
description: "Low-level project initialization and runtime management."
|
||||
---
|
||||
|
||||
`ktx dev` contains low-level project initialization and managed Python runtime
|
||||
commands. Context building lives at the root as
|
||||
[`ktx ingest`](/docs/cli-reference/ktx-ingest). Most users should start with
|
||||
`ktx setup`; use `ktx dev` when preparing local fixtures, checking the bundled
|
||||
runtime, or debugging runtime state.
|
||||
|
||||
## Command signature
|
||||
|
||||
```bash
|
||||
ktx dev <subcommand> [options]
|
||||
```
|
||||
|
||||
## Subcommands
|
||||
|
||||
| Subcommand | Description |
|
||||
|-----------|-------------|
|
||||
| `init [directory]` | Initialize a Git-backed KTX project directory for maintenance scripts |
|
||||
| `schema` | Print a JSON Schema describing `ktx.yaml` |
|
||||
| `runtime` | Install, start, stop, and inspect the KTX-managed Python runtime |
|
||||
|
||||
## `dev init`
|
||||
|
||||
| Flag | Description | Default |
|
||||
|------|-------------|---------|
|
||||
| `--force` | Rewrite `ktx.yaml` and scaffold files in an existing project | `false` |
|
||||
|
||||
## `dev schema`
|
||||
|
||||
`ktx dev schema` does not require a `ktx.yaml` file or a configured project
|
||||
directory. Use it from any directory to generate editor or agent schema files.
|
||||
|
||||
| Flag | Description | Default |
|
||||
|------|-------------|---------|
|
||||
| `--output <file>` | Write the schema to a file instead of stdout | - |
|
||||
|
||||
## `dev runtime` Subcommands
|
||||
|
||||
| Subcommand | Description |
|
||||
|-----------|-------------|
|
||||
| `install` | Install the bundled Python runtime wheel into the managed runtime |
|
||||
| `start` | Start the KTX-managed Python HTTP daemon |
|
||||
| `stop` | Stop the KTX-managed Python HTTP daemon |
|
||||
| `status` | Show managed Python runtime status and readiness checks |
|
||||
|
||||
## `dev runtime` Options
|
||||
|
||||
| Flag | Description | Default |
|
||||
|------|-------------|---------|
|
||||
| `--feature <feature>` | Runtime feature level for `install` and `start` (`core` or `local-embeddings`) | `core` |
|
||||
| `--json` | Print JSON output for `status` | `false` |
|
||||
| `--yes` | Accepted by `install` for scripted install commands | `false` |
|
||||
| `--force` | Reinstall for `install`, or restart for `start` | `false` |
|
||||
| `--all` | Stop all recorded or discoverable KTX daemon processes with `stop` | `false` |
|
||||
|
||||
## Examples
|
||||
|
||||
```bash
|
||||
ktx dev init
|
||||
ktx dev init ./my-project
|
||||
ktx dev init --force
|
||||
|
||||
ktx dev schema
|
||||
ktx dev schema --output ./ktx.schema.json
|
||||
|
||||
ktx dev runtime install --yes
|
||||
ktx dev runtime install --feature local-embeddings --yes
|
||||
ktx dev runtime status
|
||||
ktx dev runtime start
|
||||
ktx dev runtime start --feature local-embeddings
|
||||
ktx dev runtime stop
|
||||
ktx dev runtime stop --all
|
||||
```
|
||||
|
||||
## Output
|
||||
|
||||
Runtime commands print the runtime root, installed features, daemon URL, daemon
|
||||
pid, and log paths where relevant. `ktx dev runtime status --json` includes the
|
||||
runtime status plus readiness checks.
|
||||
|
||||
## Common errors
|
||||
|
||||
| Error | Cause | Recovery |
|
||||
|-------|-------|----------|
|
||||
| Runtime status reports missing pieces | Packages, Python environment, or linked CLI are not ready | Run `pnpm install`, `pnpm run setup:dev`, `uv sync --all-groups`, then `ktx dev runtime status` |
|
||||
| Runtime daemon does not start | The managed Python runtime is missing or stale | Run `ktx dev runtime install --yes`, then `ktx dev runtime start` |
|
||||
| Multiple daemon processes remain | Older daemon state files or stray processes exist | Run `ktx dev runtime stop --all`, then start the runtime again |
|
||||
|
|
@ -156,6 +156,6 @@ KTX_INGEST_TRACE_LEVEL=trace ktx ingest metabase
|
|||
| Connection not configured | The connection id is not present in `ktx.yaml` | Add the connection with `ktx setup` or update `ktx.yaml` |
|
||||
| Deep readiness is missing | `--deep` or query history needs model, embedding, and scan-enrichment configuration | Run `ktx setup` or rerun with `--fast` |
|
||||
| Query history is unsupported | The selected database driver does not support query history | Run schema ingest without query-history flags |
|
||||
| Python runtime is missing | The selected ingest target needs runtime-backed SQL analysis or source parsing | Accept the interactive prompt, rerun with `--yes`, or run the suggested `ktx dev runtime install` command |
|
||||
| Python runtime is missing | The selected ingest target needs runtime-backed SQL analysis or source parsing | Accept the interactive prompt, rerun with `--yes`, or run the suggested `ktx admin runtime install` command |
|
||||
| Source options were ignored | Depth and query-history flags were supplied for a non-database source | Omit database-only flags when ingesting source connections |
|
||||
| Text ingest stops early | `--fail-fast` was used and one item failed | Fix the failed item or rerun without `--fail-fast` to collect all failures |
|
||||
|
|
|
|||
|
|
@ -231,7 +231,7 @@ Use `ktx status` for repeatable readiness checks after setup exits.
|
|||
| Setup resumes an unexpected project | `KTX_PROJECT_DIR` or nearest `ktx.yaml` points to another directory | Pass `--project-dir <path>` explicitly |
|
||||
| Setup cannot run in CI | Required values are missing and `--no-input` disables prompts | Provide the relevant automation flags or create a fixture `ktx.yaml` |
|
||||
| Provider health check fails | Provider key, model id, Vertex project, or Vertex location is invalid | Fix the `env:` or `file:` reference and rerun setup |
|
||||
| Python runtime is missing | The selected setup needs runtime-backed agent, query-history, Looker, or local embedding features | Accept the interactive prompt, rerun with `--yes`, or run the suggested `ktx dev runtime install` command |
|
||||
| Python runtime is missing | The selected setup needs runtime-backed agent, query-history, Looker, or local embedding features | Accept the interactive prompt, rerun with `--yes`, or run the suggested `ktx admin runtime install` command |
|
||||
| `--enable-query-history` is rejected | The selected database driver does not support query history | Use Postgres, BigQuery, or Snowflake, or rerun without query-history flags |
|
||||
| Source setup rejects location flags | Both `--source-path` and `--source-git-url` were supplied | Choose the local path or the Git URL, not both |
|
||||
| Agent integration missing | Setup skipped the agents step | Run `ktx setup --agents --target <target>` |
|
||||
|
|
|
|||
|
|
@ -164,4 +164,4 @@ percentage.
|
|||
| Validation fails | YAML references missing columns, invalid joins, or invalid SQL expressions | Fix the source YAML and rerun `ktx sl validate` |
|
||||
| Query compile fails | Measure, dimension, filter, or segment name is invalid | Search sources with `ktx sl <query>`, inspect the source YAML in your project files, then retry using declared fields |
|
||||
| Execution returns too many rows | `--max-rows` is missing or too high | Add `--max-rows` with a bounded value before executing |
|
||||
| Runtime install is blocked | Query execution needs the managed Python runtime and prompts are disabled | Run `ktx dev runtime install --feature core --yes`, or rerun `ktx sl query --yes` |
|
||||
| Runtime install is blocked | Query execution needs the managed Python runtime and prompts are disabled | Run `ktx admin runtime install --feature core --yes`, or rerun `ktx sl query --yes` |
|
||||
|
|
|
|||
|
|
@ -48,7 +48,7 @@ ktx
|
|||
stop
|
||||
status
|
||||
logs
|
||||
dev
|
||||
admin
|
||||
init [directory]
|
||||
schema
|
||||
runtime
|
||||
|
|
@ -56,6 +56,7 @@ ktx
|
|||
start
|
||||
stop
|
||||
status
|
||||
reindex
|
||||
```
|
||||
|
||||
The public context-build entrypoint is `ktx ingest [connectionId]` or
|
||||
|
|
|
|||
|
|
@ -11,6 +11,6 @@
|
|||
"ktx-wiki",
|
||||
"ktx-status",
|
||||
"ktx-mcp",
|
||||
"ktx-dev"
|
||||
"ktx-admin"
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ description: How KTX compiles a short Semantic Query into safe, dialect-correct
|
|||
import { SemanticLayerFlow } from "@/components/semantic-layer-flow";
|
||||
|
||||
KTX's semantic layer is a compiler that turns intent into SQL. The agent
|
||||
declares _what_ it wants — measures, dimensions, filters — in a small
|
||||
declares _what_ it wants - measures, dimensions, filters - in a small
|
||||
Semantic Query. KTX figures out the _how_: which tables to join, what
|
||||
grain to aggregate at, how to keep fan-out from inflating measures, and
|
||||
what dialect the warehouse speaks.
|
||||
|
|
@ -21,8 +21,8 @@ This page covers four mechanics:
|
|||
## Imperative SQL vs declarative Semantic Querying
|
||||
|
||||
Writing analytics SQL is imperative work. Every question forces the
|
||||
agent to hold two things in mind at once: _what_ it wants — a measure, a
|
||||
slice, a filter — and _how_ to compute it: which tables to join, which
|
||||
agent to hold two things in mind at once: _what_ it wants - a measure, a
|
||||
slice, a filter - and _how_ to compute it: which tables to join, which
|
||||
key links them, what grain to aggregate at, how to keep one fact from
|
||||
inflating another, and what dialect the warehouse speaks. Plumbing on
|
||||
top of intent, every query.
|
||||
|
|
@ -30,7 +30,7 @@ top of intent, every query.
|
|||
KTX's semantic layer separates those concerns:
|
||||
|
||||
- **You and KTX maintain the how.** Sources, joins, grain, measures, and
|
||||
segments live in reviewable YAML — the analytical contract the team
|
||||
segments live in reviewable YAML - the analytical contract the team
|
||||
agrees on, version-controlled.
|
||||
- **The agent declares the what.** It sends a Semantic Query and trusts
|
||||
the compiler to produce safe SQL.
|
||||
|
|
|
|||
|
|
@ -126,8 +126,8 @@ If you choose local `sentence-transformers` embeddings, KTX uses the managed
|
|||
Python runtime. To prepare it before setup, run:
|
||||
|
||||
```bash
|
||||
ktx dev runtime install --feature local-embeddings --yes
|
||||
ktx dev runtime start --feature local-embeddings
|
||||
ktx admin runtime install --feature local-embeddings --yes
|
||||
ktx admin runtime start --feature local-embeddings
|
||||
```
|
||||
|
||||
During the database step, setup tests the saved connection and builds initial
|
||||
|
|
|
|||
|
|
@ -12,8 +12,8 @@ imports the package entry point, and runs installed `ktx` commands against a
|
|||
generated local project.
|
||||
|
||||
The managed Python runtime smoke requires `uv` on `PATH`, isolates
|
||||
`KTX_RUNTIME_ROOT`, verifies `ktx dev runtime status`, runs `ktx sl query --yes` to
|
||||
install the core runtime from the bundled wheel, checks `ktx dev runtime status`,
|
||||
`KTX_RUNTIME_ROOT`, verifies `ktx admin runtime status`, runs `ktx sl query --yes` to
|
||||
install the core runtime from the bundled wheel, checks `ktx admin runtime status`,
|
||||
starts and reuses the managed daemon, and stops it.
|
||||
|
||||
The artifact manifest contains the public `@kaelio/ktx` npm tarball and the
|
||||
|
|
|
|||
145
packages/cli/src/admin-reindex.test.ts
Normal file
145
packages/cli/src/admin-reindex.test.ts
Normal file
|
|
@ -0,0 +1,145 @@
|
|||
import type { ReindexSummary } from '@ktx/context/index-sync';
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
import { renderReindexJson, renderReindexPlain, reindexHasErrors } from './admin-reindex.js';
|
||||
import { runKtxCli } from './index.js';
|
||||
|
||||
function makeIo(options: { stdoutIsTTY?: boolean } = {}) {
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
return {
|
||||
io: {
|
||||
stdout: {
|
||||
isTTY: options.stdoutIsTTY,
|
||||
write: (chunk: string) => {
|
||||
stdout += chunk;
|
||||
},
|
||||
},
|
||||
stderr: {
|
||||
write: (chunk: string) => {
|
||||
stderr += chunk;
|
||||
},
|
||||
},
|
||||
},
|
||||
stdout: () => stdout,
|
||||
stderr: () => stderr,
|
||||
};
|
||||
}
|
||||
|
||||
function summary(overrides: Partial<ReindexSummary> = {}): ReindexSummary {
|
||||
return {
|
||||
scopes: [
|
||||
{
|
||||
kind: 'wiki',
|
||||
label: 'global',
|
||||
scope: 'global',
|
||||
scopeId: null,
|
||||
scanned: 42,
|
||||
updated: 3,
|
||||
deleted: 1,
|
||||
embeddingsRecomputed: 3,
|
||||
embeddingsFailed: 0,
|
||||
durationMs: 412,
|
||||
},
|
||||
{
|
||||
kind: 'sl',
|
||||
label: 'warehouse',
|
||||
connectionId: 'warehouse',
|
||||
scanned: 18,
|
||||
updated: 2,
|
||||
deleted: 0,
|
||||
embeddingsRecomputed: 2,
|
||||
embeddingsFailed: 0,
|
||||
durationMs: 287,
|
||||
},
|
||||
],
|
||||
totals: { scanned: 60, updated: 5, deleted: 1, embeddingsRecomputed: 5, embeddingsFailed: 0 },
|
||||
dbPath: '.ktx/db.sqlite',
|
||||
force: false,
|
||||
embeddingsAvailable: true,
|
||||
durationMs: 1234,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe('admin reindex renderers', () => {
|
||||
it('renders plain scope lines to stderr and summary to stdout', () => {
|
||||
const io = makeIo();
|
||||
|
||||
renderReindexPlain(summary(), io.io);
|
||||
|
||||
expect(io.stderr()).toContain('wiki/global\tscanned=42\tupdated=3\tdeleted=1\tembeddings=3\tduration_ms=412\n');
|
||||
expect(io.stderr()).toContain('sl/warehouse\tscanned=18\tupdated=2\tdeleted=0\tembeddings=2\tduration_ms=287\n');
|
||||
expect(io.stdout()).toBe('reindex\tscopes=2\tscanned=60\tupdated=5\tdeleted=1\tembeddings=5\tduration_ms=1234\n');
|
||||
});
|
||||
|
||||
it('renders rebuilt labels in plain force mode', () => {
|
||||
const io = makeIo();
|
||||
|
||||
renderReindexPlain(summary({ force: true }), io.io);
|
||||
|
||||
expect(io.stderr()).toContain('rebuilt=3');
|
||||
expect(io.stdout()).toContain('rebuilt=5');
|
||||
expect(io.stdout()).not.toContain('updated=5');
|
||||
});
|
||||
|
||||
it('renders json envelope to stdout only', () => {
|
||||
const io = makeIo();
|
||||
|
||||
renderReindexJson(summary(), io.io);
|
||||
|
||||
expect(JSON.parse(io.stdout())).toMatchObject({
|
||||
kind: 'reindex',
|
||||
data: { totals: { scanned: 60, updated: 5 } },
|
||||
meta: { command: 'admin reindex' },
|
||||
});
|
||||
expect(io.stderr()).toBe('');
|
||||
});
|
||||
|
||||
it('detects per-scope errors', () => {
|
||||
expect(
|
||||
reindexHasErrors(
|
||||
summary({
|
||||
scopes: [{ ...summary().scopes[0]!, error: 'provider failed' }],
|
||||
}),
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('admin reindex Commander routing', () => {
|
||||
it('routes flags to the injectable reindex runner', async () => {
|
||||
const { mkdir, mkdtemp, rm, writeFile } = await import('node:fs/promises');
|
||||
const { tmpdir } = await import('node:os');
|
||||
const { join } = await import('node:path');
|
||||
const tempDir = await mkdtemp(join(tmpdir(), 'ktx-admin-reindex-cli-'));
|
||||
const projectDir = join(tempDir, 'project');
|
||||
const io = makeIo();
|
||||
const adminReindex = vi.fn(async () => 0);
|
||||
|
||||
try {
|
||||
await mkdir(projectDir, { recursive: true });
|
||||
await writeFile(join(projectDir, 'ktx.yaml'), '{}\n', 'utf-8');
|
||||
|
||||
await expect(
|
||||
runKtxCli(
|
||||
['--project-dir', projectDir, 'admin', 'reindex', '--force', '--json', '--output', 'plain'],
|
||||
io.io,
|
||||
{ adminReindex },
|
||||
),
|
||||
).resolves.toBe(0);
|
||||
} finally {
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
expect(adminReindex).toHaveBeenCalledWith(
|
||||
{
|
||||
projectDir,
|
||||
force: true,
|
||||
json: true,
|
||||
output: 'plain',
|
||||
cliVersion: '0.1.0-rc.1',
|
||||
},
|
||||
io.io,
|
||||
);
|
||||
});
|
||||
});
|
||||
210
packages/cli/src/admin-reindex.ts
Normal file
210
packages/cli/src/admin-reindex.ts
Normal file
|
|
@ -0,0 +1,210 @@
|
|||
import {
|
||||
createLocalKtxEmbeddingProviderFromConfig,
|
||||
KtxIngestEmbeddingPortAdapter,
|
||||
MANAGED_SENTENCE_TRANSFORMERS_BASE_URL,
|
||||
type KtxEmbeddingPort,
|
||||
} from '@ktx/context';
|
||||
import { reindexLocalIndexes, type ReindexScopeResult, type ReindexSummary } from '@ktx/context/index-sync';
|
||||
import { loadKtxProject, type KtxLocalProject } from '@ktx/context/project';
|
||||
import { Option, type Command } from '@commander-js/extra-typings';
|
||||
import { cancel, intro, log, note, outro } from '@clack/prompts';
|
||||
import type { KtxCliCommandContext } from './cli-program.js';
|
||||
import type { KtxCliIo } from './cli-runtime.js';
|
||||
import { resolveOutputMode } from './io/mode.js';
|
||||
import { green, red, SYMBOLS } from './io/symbols.js';
|
||||
import { ensureManagedLocalEmbeddingsDaemon } from './managed-local-embeddings.js';
|
||||
|
||||
export interface KtxAdminReindexArgs {
|
||||
projectDir: string;
|
||||
force: boolean;
|
||||
output?: 'pretty' | 'plain' | 'json';
|
||||
json?: boolean;
|
||||
cliVersion: string;
|
||||
}
|
||||
|
||||
export function registerAdminReindexCommand(admin: Command, context: KtxCliCommandContext): void {
|
||||
admin
|
||||
.command('reindex')
|
||||
.description('Sync local wiki and semantic-layer search indexes from disk')
|
||||
.option('--force', 'Clear each discovered scope before rebuilding it', false)
|
||||
.option('--json', 'Shortcut for --output=json (overrides --output)', false)
|
||||
.addOption(
|
||||
new Option('--output <mode>', 'Output mode: pretty, plain, or json').choices(['pretty', 'plain', 'json']),
|
||||
)
|
||||
.action(async (options: { force?: boolean; json?: boolean; output?: 'pretty' | 'plain' | 'json' }, command) => {
|
||||
const runner = context.deps.adminReindex ?? runKtxAdminReindex;
|
||||
const { resolveCommandProjectDir } = await import('./cli-program.js');
|
||||
context.setExitCode(
|
||||
await runner(
|
||||
{
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
force: options.force === true,
|
||||
json: options.json === true,
|
||||
output: options.output,
|
||||
cliVersion: context.packageInfo.version,
|
||||
},
|
||||
context.io,
|
||||
),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
async function resolveReindexEmbeddingService(
|
||||
project: KtxLocalProject,
|
||||
args: KtxAdminReindexArgs,
|
||||
io: KtxCliIo,
|
||||
): Promise<KtxEmbeddingPort | null> {
|
||||
const config = project.config.ingest.embeddings;
|
||||
if (config.backend === 'none') {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (
|
||||
config.backend === 'sentence-transformers' &&
|
||||
config.sentenceTransformers?.base_url === MANAGED_SENTENCE_TRANSFORMERS_BASE_URL
|
||||
) {
|
||||
const daemon = await ensureManagedLocalEmbeddingsDaemon({
|
||||
cliVersion: args.cliVersion,
|
||||
projectDir: project.projectDir,
|
||||
installPolicy: 'never',
|
||||
io,
|
||||
});
|
||||
const provider = createLocalKtxEmbeddingProviderFromConfig(config, { env: { ...process.env, ...daemon.env } });
|
||||
return provider ? new KtxIngestEmbeddingPortAdapter(provider) : null;
|
||||
}
|
||||
|
||||
const provider = createLocalKtxEmbeddingProviderFromConfig(config);
|
||||
return provider ? new KtxIngestEmbeddingPortAdapter(provider) : null;
|
||||
}
|
||||
|
||||
function scopeKey(scope: ReindexScopeResult): string {
|
||||
if (scope.kind === 'wiki') {
|
||||
return scope.scope === 'user' ? `wiki/user/${scope.scopeId ?? 'local'}` : 'wiki/global';
|
||||
}
|
||||
return `sl/${scope.connectionId ?? scope.label}`;
|
||||
}
|
||||
|
||||
function quotePlainValue(value: string): string {
|
||||
return `"${value.replaceAll('\\', '\\\\').replaceAll('"', '\\"')}"`;
|
||||
}
|
||||
|
||||
export function reindexHasErrors(summary: ReindexSummary): boolean {
|
||||
return summary.scopes.some((scope) => scope.error);
|
||||
}
|
||||
|
||||
export function renderReindexPlain(summary: ReindexSummary, io: KtxCliIo): void {
|
||||
const updateKey = summary.force ? 'rebuilt' : 'updated';
|
||||
for (const scope of summary.scopes) {
|
||||
const cells = [
|
||||
scopeKey(scope),
|
||||
`scanned=${scope.scanned}`,
|
||||
`${updateKey}=${scope.updated}`,
|
||||
`deleted=${scope.deleted}`,
|
||||
`embeddings=${summary.embeddingsAvailable ? String(scope.embeddingsRecomputed) : '-'}`,
|
||||
`duration_ms=${scope.durationMs}`,
|
||||
...(scope.error ? [`error=${quotePlainValue(scope.error)}`] : []),
|
||||
];
|
||||
io.stderr.write(`${cells.join('\t')}\n`);
|
||||
}
|
||||
const failed = summary.scopes.filter((scope) => scope.error).length;
|
||||
io.stdout.write(
|
||||
[
|
||||
'reindex',
|
||||
`scopes=${summary.scopes.length}`,
|
||||
`scanned=${summary.totals.scanned}`,
|
||||
`${updateKey}=${summary.totals.updated}`,
|
||||
`deleted=${summary.totals.deleted}`,
|
||||
`embeddings=${summary.embeddingsAvailable ? String(summary.totals.embeddingsRecomputed) : '-'}`,
|
||||
`duration_ms=${summary.durationMs}`,
|
||||
...(failed > 0 ? [`failed=${failed}`] : []),
|
||||
].join('\t') + '\n',
|
||||
);
|
||||
}
|
||||
|
||||
export function renderReindexJson(summary: ReindexSummary, io: KtxCliIo): void {
|
||||
io.stdout.write(`${JSON.stringify({ kind: 'reindex', data: summary, meta: { command: 'admin reindex' } }, null, 2)}\n`);
|
||||
}
|
||||
|
||||
function noun(scope: ReindexScopeResult): string {
|
||||
return scope.kind === 'wiki' ? 'pages' : 'sources';
|
||||
}
|
||||
|
||||
function formatScopeLine(scope: ReindexScopeResult, force: boolean, embeddingsAvailable: boolean): string {
|
||||
if (scope.error) {
|
||||
return `${scope.kind === 'wiki' ? 'Wiki' : 'SL'}: ${scope.label} ${SYMBOLS.emDash} failed: ${scope.error}`;
|
||||
}
|
||||
const changedLabel = force ? 'rebuilt' : 'updated';
|
||||
const parts = [`${scope.scanned} ${noun(scope)}`];
|
||||
if (scope.updated > 0) {
|
||||
parts.push(`${scope.updated} ${changedLabel}`);
|
||||
} else {
|
||||
parts.push('unchanged');
|
||||
}
|
||||
if (!force && scope.deleted > 0) {
|
||||
parts.push(`${scope.deleted} deleted`);
|
||||
}
|
||||
if (embeddingsAvailable) {
|
||||
parts.push(`${scope.embeddingsRecomputed} embeddings recomputed`);
|
||||
}
|
||||
parts.push(`${scope.durationMs}ms`);
|
||||
return `${scope.kind === 'wiki' ? 'Wiki' : 'SL'}: ${scope.label} ${SYMBOLS.emDash} ${parts.join(` ${SYMBOLS.middot} `)}`;
|
||||
}
|
||||
|
||||
function renderReindexPretty(summary: ReindexSummary, io: KtxCliIo): void {
|
||||
intro(summary.force ? 'ktx admin reindex --force' : 'ktx admin reindex');
|
||||
if (!summary.embeddingsAvailable) {
|
||||
log.warn(`Embeddings: not configured ${SYMBOLS.emDash} indexing lexical only`);
|
||||
}
|
||||
for (const scope of summary.scopes) {
|
||||
const line = formatScopeLine(scope, summary.force, summary.embeddingsAvailable);
|
||||
if (scope.error) {
|
||||
log.error(red(line));
|
||||
} else {
|
||||
log.success(green(line));
|
||||
}
|
||||
}
|
||||
const failed = summary.scopes.filter((scope) => scope.error).length;
|
||||
note(
|
||||
[
|
||||
`scopes ${summary.scopes.length}`,
|
||||
`scanned ${summary.totals.scanned}`,
|
||||
`${summary.force ? 'rebuilt' : 'updated'} ${summary.totals.updated}`,
|
||||
`deleted ${summary.totals.deleted}`,
|
||||
`embeddings ${summary.embeddingsAvailable ? summary.totals.embeddingsRecomputed : SYMBOLS.emDash}`,
|
||||
`index ${summary.dbPath}`,
|
||||
...(failed > 0 ? [`failed ${failed}`] : []),
|
||||
].join('\n'),
|
||||
'Summary',
|
||||
);
|
||||
if (failed > 0) {
|
||||
cancel(`reindex completed with ${failed} error${failed === 1 ? '' : 's'}`);
|
||||
} else {
|
||||
outro(`Done in ${(summary.durationMs / 1000).toFixed(1)}s`);
|
||||
}
|
||||
void io;
|
||||
}
|
||||
|
||||
async function runKtxAdminReindex(args: KtxAdminReindexArgs, io: KtxCliIo = process): Promise<number> {
|
||||
try {
|
||||
const project = await loadKtxProject({ projectDir: args.projectDir });
|
||||
const embeddingService = await resolveReindexEmbeddingService(project, args, io);
|
||||
const summary = await reindexLocalIndexes(project, { force: args.force, embeddingService });
|
||||
const mode = resolveOutputMode({ explicit: args.output, json: args.json, io });
|
||||
|
||||
if (!summary.embeddingsAvailable && mode === 'plain') {
|
||||
io.stderr.write(`Embeddings: not configured ${SYMBOLS.emDash} indexing lexical only\n`);
|
||||
}
|
||||
|
||||
if (mode === 'json') {
|
||||
renderReindexJson(summary, io);
|
||||
} else if (mode === 'plain') {
|
||||
renderReindexPlain(summary, io);
|
||||
} else {
|
||||
renderReindexPretty(summary, io);
|
||||
}
|
||||
return reindexHasErrors(summary) ? 1 : 0;
|
||||
} catch (error) {
|
||||
io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
|
@ -22,14 +22,14 @@ function makeIo() {
|
|||
};
|
||||
}
|
||||
|
||||
describe('dev Commander tree', () => {
|
||||
it('prints visible dev help with only supported low-level command groups', async () => {
|
||||
describe('admin Commander tree', () => {
|
||||
it('prints visible admin help with supported low-level command groups', async () => {
|
||||
const testIo = makeIo();
|
||||
|
||||
await expect(runKtxCli(['dev', '--help'], testIo.io)).resolves.toBe(0);
|
||||
await expect(runKtxCli(['admin', '--help'], testIo.io)).resolves.toBe(0);
|
||||
|
||||
expect(testIo.stdout()).toContain('Usage: ktx dev [options] [command]');
|
||||
for (const command of ['init', 'runtime']) {
|
||||
expect(testIo.stdout()).toContain('Usage: ktx admin [options] [command]');
|
||||
for (const command of ['init', 'runtime', 'reindex']) {
|
||||
expect(testIo.stdout()).toContain(command);
|
||||
}
|
||||
for (const removed of [
|
||||
|
|
@ -52,27 +52,35 @@ describe('dev Commander tree', () => {
|
|||
expect(testIo.stderr()).toBe('');
|
||||
});
|
||||
|
||||
it('lists dev in root command rows', async () => {
|
||||
it('lists admin in root command rows', async () => {
|
||||
const testIo = makeIo();
|
||||
|
||||
await expect(runKtxCli(['--help'], testIo.io)).resolves.toBe(0);
|
||||
|
||||
expect(testIo.stdout()).not.toContain('Advanced:');
|
||||
expect(testIo.stdout()).toContain('dev');
|
||||
expect(testIo.stdout()).toMatch(/Low-level project initialization and runtime\s+management/);
|
||||
expect(testIo.stdout()).toContain('admin');
|
||||
expect(testIo.stdout()).toMatch(/Low-level project initialization,\s+runtime,\s+and index management/);
|
||||
expect(testIo.stderr()).toBe('');
|
||||
});
|
||||
|
||||
it('keeps project scaffolding under dev init', async () => {
|
||||
it('does not keep a dev alias', async () => {
|
||||
const testIo = makeIo();
|
||||
|
||||
await expect(runKtxCli(['dev', '--help'], testIo.io)).resolves.toBe(1);
|
||||
|
||||
expect(testIo.stderr()).toContain("unknown command 'dev'");
|
||||
});
|
||||
|
||||
it('keeps project scaffolding under admin init', async () => {
|
||||
const { mkdtemp, readFile, rm } = await import('node:fs/promises');
|
||||
const { tmpdir } = await import('node:os');
|
||||
const { join } = await import('node:path');
|
||||
const tempDir = await mkdtemp(join(tmpdir(), 'ktx-dev-init-'));
|
||||
const tempDir = await mkdtemp(join(tmpdir(), 'ktx-admin-init-'));
|
||||
const projectDir = join(tempDir, 'warehouse');
|
||||
const testIo = makeIo();
|
||||
|
||||
try {
|
||||
await expect(runKtxCli(['dev', 'init', projectDir], testIo.io)).resolves.toBe(0);
|
||||
await expect(runKtxCli(['admin', 'init', projectDir], testIo.io)).resolves.toBe(0);
|
||||
|
||||
expect(testIo.stdout()).toContain(`Initialized KTX project at ${projectDir}`);
|
||||
await expect(readFile(join(projectDir, 'ktx.yaml'), 'utf-8')).resolves.not.toContain('project:');
|
||||
|
|
@ -82,17 +90,17 @@ describe('dev Commander tree', () => {
|
|||
}
|
||||
});
|
||||
|
||||
it('uses global project-dir for dev init when the positional directory is omitted', async () => {
|
||||
it('uses global project-dir for admin init when the positional directory is omitted', async () => {
|
||||
const { mkdtemp, rm } = await import('node:fs/promises');
|
||||
const { tmpdir } = await import('node:os');
|
||||
const { join } = await import('node:path');
|
||||
const tempDir = await mkdtemp(join(tmpdir(), 'ktx-dev-init-global-'));
|
||||
const tempDir = await mkdtemp(join(tmpdir(), 'ktx-admin-init-global-'));
|
||||
const projectDir = join(tempDir, 'global-init');
|
||||
const testIo = makeIo();
|
||||
|
||||
try {
|
||||
await expect(
|
||||
runKtxCli(['--project-dir', projectDir, 'dev', 'init'], testIo.io),
|
||||
runKtxCli(['--project-dir', projectDir, 'admin', 'init'], testIo.io),
|
||||
).resolves.toBe(0);
|
||||
|
||||
expect(testIo.stdout()).toContain(`Initialized KTX project at ${projectDir}`);
|
||||
|
|
@ -106,7 +114,7 @@ describe('dev Commander tree', () => {
|
|||
const { mkdtemp, rm } = await import('node:fs/promises');
|
||||
const { tmpdir } = await import('node:os');
|
||||
const { join } = await import('node:path');
|
||||
const tempDir = await mkdtemp(join(tmpdir(), 'ktx-dev-schema-'));
|
||||
const tempDir = await mkdtemp(join(tmpdir(), 'ktx-admin-schema-'));
|
||||
const missingProjectDir = join(tempDir, 'missing-project');
|
||||
const originalProjectDir = process.env.KTX_PROJECT_DIR;
|
||||
const testIo = makeIo();
|
||||
|
|
@ -114,7 +122,7 @@ describe('dev Commander tree', () => {
|
|||
try {
|
||||
process.env.KTX_PROJECT_DIR = missingProjectDir;
|
||||
|
||||
await expect(runKtxCli(['dev', 'schema'], testIo.io)).resolves.toBe(0);
|
||||
await expect(runKtxCli(['admin', 'schema'], testIo.io)).resolves.toBe(0);
|
||||
|
||||
expect(JSON.parse(testIo.stdout())).toMatchObject({
|
||||
title: 'ktx.yaml',
|
||||
|
|
@ -131,19 +139,19 @@ describe('dev Commander tree', () => {
|
|||
}
|
||||
});
|
||||
|
||||
it('rejects removed dev command groups', async () => {
|
||||
it('rejects removed admin command groups', async () => {
|
||||
for (const argv of [
|
||||
['dev', 'doctor', 'setup'],
|
||||
['dev', 'runtime', 'doctor'],
|
||||
['dev', 'runtime', 'prune', '--dry-run'],
|
||||
['dev', 'scan', 'warehouse'],
|
||||
['dev', 'ingest', 'run'],
|
||||
['dev', 'mapping', 'list'],
|
||||
['dev', 'completion', 'zsh'],
|
||||
['dev', '__complete', '--shell', 'zsh', '--position', '2', '--', 'ktx', ''],
|
||||
['dev', 'knowledge', 'list'],
|
||||
['dev', 'model', 'list'],
|
||||
['dev', 'artifacts'],
|
||||
['admin', 'doctor', 'setup'],
|
||||
['admin', 'runtime', 'doctor'],
|
||||
['admin', 'runtime', 'prune', '--dry-run'],
|
||||
['admin', 'scan', 'warehouse'],
|
||||
['admin', 'ingest', 'run'],
|
||||
['admin', 'mapping', 'list'],
|
||||
['admin', 'completion', 'zsh'],
|
||||
['admin', '__complete', '--shell', 'zsh', '--position', '2', '--', 'ktx', ''],
|
||||
['admin', 'knowledge', 'list'],
|
||||
['admin', 'model', 'list'],
|
||||
['admin', 'artifacts'],
|
||||
]) {
|
||||
const testIo = makeIo();
|
||||
|
||||
|
|
@ -155,8 +163,8 @@ describe('dev Commander tree', () => {
|
|||
|
||||
it.each([
|
||||
{
|
||||
argv: ['dev', 'runtime', '--help'],
|
||||
expected: ['Usage: ktx dev runtime', 'install', 'start', 'stop', 'status'],
|
||||
argv: ['admin', 'runtime', '--help'],
|
||||
expected: ['Usage: ktx admin runtime', 'install', 'start', 'stop', 'status'],
|
||||
},
|
||||
])('prints generated nested help for $argv', async ({ argv, expected }) => {
|
||||
const io = makeIo();
|
||||
|
|
@ -167,7 +175,7 @@ describe('dev Commander tree', () => {
|
|||
for (const text of expected) {
|
||||
expect(io.stdout()).toContain(text);
|
||||
}
|
||||
if (argv.join(' ') === 'dev runtime --help') {
|
||||
if (argv.join(' ') === 'admin runtime --help') {
|
||||
expect(io.stdout()).not.toContain('prune');
|
||||
expect(io.stdout()).not.toContain('doctor');
|
||||
}
|
||||
|
|
@ -1,27 +1,28 @@
|
|||
import { resolve } from 'node:path';
|
||||
import type { Command } from '@commander-js/extra-typings';
|
||||
import { type CommandWithGlobalOptions, type KtxCliCommandContext, resolveCommandProjectDir } from './cli-program.js';
|
||||
import { registerAdminReindexCommand } from './admin-reindex.js';
|
||||
import { registerRuntimeCommands } from './commands/runtime-commands.js';
|
||||
import { profileMark } from './startup-profile.js';
|
||||
|
||||
profileMark('module:dev');
|
||||
profileMark('module:admin');
|
||||
|
||||
export function registerDevCommands(program: Command, context: KtxCliCommandContext): void {
|
||||
const dev = program
|
||||
.command('dev')
|
||||
.description('Low-level project initialization and runtime management')
|
||||
export function registerAdminCommands(program: Command, context: KtxCliCommandContext): void {
|
||||
const admin = program
|
||||
.command('admin')
|
||||
.description('Low-level project initialization, runtime, and index management')
|
||||
.showHelpAfterError();
|
||||
|
||||
dev.hook('preAction', (_thisCommand, actionCommand) => {
|
||||
context.writeDebug?.('dev', actionCommand);
|
||||
admin.hook('preAction', (_thisCommand, actionCommand) => {
|
||||
context.writeDebug?.('admin', actionCommand);
|
||||
});
|
||||
|
||||
dev.action(() => {
|
||||
dev.outputHelp();
|
||||
admin.action(() => {
|
||||
admin.outputHelp();
|
||||
context.setExitCode(0);
|
||||
});
|
||||
|
||||
dev
|
||||
admin
|
||||
.command('init')
|
||||
.description('Initialize a Git-backed KTX project directory for maintenance scripts')
|
||||
.argument('[directory]', 'Project directory')
|
||||
|
|
@ -44,7 +45,7 @@ export function registerDevCommands(program: Command, context: KtxCliCommandCont
|
|||
},
|
||||
);
|
||||
|
||||
dev
|
||||
admin
|
||||
.command('schema')
|
||||
.description('Print a JSON Schema describing ktx.yaml (for editors and LLM agents)')
|
||||
.option('--output <file>', 'Write the schema to a file instead of stdout')
|
||||
|
|
@ -62,5 +63,6 @@ export function registerDevCommands(program: Command, context: KtxCliCommandCont
|
|||
context.setExitCode(0);
|
||||
});
|
||||
|
||||
registerRuntimeCommands(dev, context);
|
||||
registerRuntimeCommands(admin, context);
|
||||
registerAdminReindexCommand(admin, context);
|
||||
}
|
||||
|
|
@ -31,7 +31,7 @@ describe('buildKtxProgram', () => {
|
|||
|
||||
expect(program.name()).toBe('ktx');
|
||||
const topLevel = program.commands.map((command) => command.name()).sort();
|
||||
for (const expected of ['setup', 'connection', 'ingest', 'sl', 'dev']) {
|
||||
for (const expected of ['setup', 'connection', 'ingest', 'sl', 'admin']) {
|
||||
expect(topLevel).toContain(expected);
|
||||
}
|
||||
});
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import { registerSetupCommands } from './commands/setup-commands.js';
|
|||
import { registerSlCommands } from './commands/sl-commands.js';
|
||||
import { registerSqlCommands } from './commands/sql-commands.js';
|
||||
import { registerStatusCommands } from './commands/status-commands.js';
|
||||
import { registerDevCommands } from './dev.js';
|
||||
import { registerAdminCommands } from './admin.js';
|
||||
import { renderMissingProjectMessage } from './doctor.js';
|
||||
import { findNearestKtxProjectDir, resolveKtxProjectDir } from './project-resolver.js';
|
||||
import { profileMark, profileSpan } from './startup-profile.js';
|
||||
|
|
@ -58,8 +58,8 @@ type CommandPathNode = CommandWithGlobalOptions & {
|
|||
};
|
||||
|
||||
const PROJECT_AWARE_ROOT_COMMANDS = new Set(['setup', 'connection', 'ingest', 'wiki', 'sl', 'sql', 'status', 'mcp']);
|
||||
const PROJECT_INDEPENDENT_DEV_COMMANDS = new Set(['runtime', 'schema']);
|
||||
const COMMANDS_THAT_CREATE_PROJECT = new Set(['setup', 'ktx dev init']);
|
||||
const PROJECT_INDEPENDENT_ADMIN_COMMANDS = new Set(['runtime', 'schema']);
|
||||
const COMMANDS_THAT_CREATE_PROJECT = new Set(['setup', 'ktx admin init']);
|
||||
const COMMANDS_WITH_OWN_MISSING_PROJECT_HANDLING = new Set(['status']);
|
||||
const GLOBAL_OPTIONS_WITH_VALUE = new Set(['--project-dir']);
|
||||
const GLOBAL_OPTIONS_WITHOUT_VALUE = new Set(['--debug', '--help', '-h', '--version', '-v']);
|
||||
|
|
@ -172,15 +172,15 @@ function isProjectAwareCommand(path: string[]): boolean {
|
|||
}
|
||||
|
||||
const rootCommand = path[1];
|
||||
if (rootCommand === 'dev') {
|
||||
return path[2] !== undefined && !PROJECT_INDEPENDENT_DEV_COMMANDS.has(path[2]);
|
||||
if (rootCommand === 'admin') {
|
||||
return path[2] !== undefined && !PROJECT_INDEPENDENT_ADMIN_COMMANDS.has(path[2]);
|
||||
}
|
||||
return rootCommand !== undefined && PROJECT_AWARE_ROOT_COMMANDS.has(rootCommand);
|
||||
}
|
||||
|
||||
function shouldSuppressProjectDirLine(path: string[], options: Record<string, unknown>): boolean {
|
||||
const commandPathKey = path.join(' ');
|
||||
if (commandPathKey === 'ktx dev init') {
|
||||
if (commandPathKey === 'ktx admin init') {
|
||||
return true;
|
||||
}
|
||||
|
||||
|
|
@ -421,7 +421,7 @@ export function buildKtxProgram(options: BuildKtxProgramOptions): Command {
|
|||
registerSqlCommands(program, context);
|
||||
registerStatusCommands(program, context);
|
||||
registerMcpCommands(program, context);
|
||||
registerDevCommands(program, context);
|
||||
registerAdminCommands(program, context);
|
||||
|
||||
return program;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import { createRequire } from 'node:module';
|
||||
|
||||
import type { KtxConnectionArgs } from './connection.js';
|
||||
import type { KtxAdminReindexArgs } from './admin-reindex.js';
|
||||
import type { KtxDoctorArgs } from './doctor.js';
|
||||
import type { KtxKnowledgeArgs } from './knowledge.js';
|
||||
import type { KtxPublicIngestArgs } from './public-ingest.js';
|
||||
|
|
@ -30,6 +31,7 @@ export interface KtxCliIo {
|
|||
}
|
||||
|
||||
export interface KtxCliDeps {
|
||||
adminReindex?: (args: KtxAdminReindexArgs, io: KtxCliIo) => Promise<number>;
|
||||
setup?: (args: KtxSetupArgs, io: KtxCliIo) => Promise<number>;
|
||||
connection?: (args: KtxConnectionArgs, io: KtxCliIo) => Promise<number>;
|
||||
doctor?: (args: KtxDoctorArgs, io: KtxCliIo) => Promise<number>;
|
||||
|
|
|
|||
|
|
@ -129,9 +129,10 @@ describe('runKtxCli', () => {
|
|||
|
||||
expect(testIo.stdout()).toContain('Usage: ktx [options] [command]');
|
||||
expect(testIo.stdout()).toContain('KTX data agent context layer CLI');
|
||||
for (const command of ['setup', 'connection', 'ingest', 'wiki', 'sl', 'status', 'dev']) {
|
||||
for (const command of ['setup', 'connection', 'ingest', 'wiki', 'sl', 'status', 'admin']) {
|
||||
expect(testIo.stdout()).toContain(`${command}`);
|
||||
}
|
||||
expect(testIo.stdout()).not.toMatch(/^ dev\s/m);
|
||||
expect(testIo.stdout()).not.toMatch(/^ scan\s/m);
|
||||
for (const removed of ['demo', 'init', 'connect', 'ask', 'knowledge', 'agent', 'completion', 'serve']) {
|
||||
expect(testIo.stdout()).not.toMatch(new RegExp(`^\\s+${removed}(?:\\s|\\[|$)`, 'm'));
|
||||
|
|
@ -291,17 +292,17 @@ describe('runKtxCli', () => {
|
|||
const pruneIo = makeIo();
|
||||
|
||||
await expect(
|
||||
runKtxCli(['dev', 'runtime', 'install', '--feature', 'local-embeddings', '--force', '--yes'], installIo.io, {
|
||||
runKtxCli(['admin', 'runtime', 'install', '--feature', 'local-embeddings', '--force', '--yes'], installIo.io, {
|
||||
runtime,
|
||||
}),
|
||||
).resolves.toBe(0);
|
||||
await expect(
|
||||
runKtxCli(['dev', 'runtime', 'start', '--feature', 'local-embeddings', '--force'], startIo.io, { runtime }),
|
||||
runKtxCli(['admin', 'runtime', 'start', '--feature', 'local-embeddings', '--force'], startIo.io, { runtime }),
|
||||
).resolves.toBe(0);
|
||||
await expect(runKtxCli(['dev', 'runtime', 'stop'], stopIo.io, { runtime })).resolves.toBe(0);
|
||||
await expect(runKtxCli(['dev', 'runtime', 'stop', '--all'], stopAllIo.io, { runtime })).resolves.toBe(0);
|
||||
await expect(runKtxCli(['dev', 'runtime', 'status', '--json'], statusIo.io, { runtime })).resolves.toBe(0);
|
||||
await expect(runKtxCli(['dev', 'runtime', 'prune', '--dry-run'], pruneIo.io, { runtime })).resolves.toBe(1);
|
||||
await expect(runKtxCli(['admin', 'runtime', 'stop'], stopIo.io, { runtime })).resolves.toBe(0);
|
||||
await expect(runKtxCli(['admin', 'runtime', 'stop', '--all'], stopAllIo.io, { runtime })).resolves.toBe(0);
|
||||
await expect(runKtxCli(['admin', 'runtime', 'status', '--json'], statusIo.io, { runtime })).resolves.toBe(0);
|
||||
await expect(runKtxCli(['admin', 'runtime', 'prune', '--dry-run'], pruneIo.io, { runtime })).resolves.toBe(1);
|
||||
|
||||
expect(runtime).toHaveBeenNthCalledWith(
|
||||
1,
|
||||
|
|
@ -402,7 +403,7 @@ describe('runKtxCli', () => {
|
|||
it('documents runtime stop all in command help', async () => {
|
||||
const testIo = makeIo();
|
||||
|
||||
await expect(runKtxCli(['dev', 'runtime', 'stop', '--help'], testIo.io)).resolves.toBe(0);
|
||||
await expect(runKtxCli(['admin', 'runtime', 'stop', '--help'], testIo.io)).resolves.toBe(0);
|
||||
|
||||
expect(testIo.stdout()).toContain('--all');
|
||||
expect(testIo.stdout()).toContain('Stop all KTX daemon processes recorded or discoverable');
|
||||
|
|
@ -680,9 +681,9 @@ describe('runKtxCli', () => {
|
|||
const completionIo = makeIo();
|
||||
const hiddenIo = makeIo();
|
||||
|
||||
await expect(runKtxCli(['dev', 'completion', 'zsh'], completionIo.io)).resolves.toBe(1);
|
||||
await expect(runKtxCli(['admin', 'completion', 'zsh'], completionIo.io)).resolves.toBe(1);
|
||||
await expect(
|
||||
runKtxCli(['dev', '__complete', '--shell', 'zsh', '--position', '2', '--', 'ktx', 'co'], hiddenIo.io),
|
||||
runKtxCli(['admin', '__complete', '--shell', 'zsh', '--position', '2', '--', 'ktx', 'co'], hiddenIo.io),
|
||||
).resolves.toBe(1);
|
||||
|
||||
expect(completionIo.stderr()).toMatch(/unknown command|error:/);
|
||||
|
|
@ -986,7 +987,7 @@ describe('runKtxCli', () => {
|
|||
expect(args.targetConnectionId).toBeUndefined();
|
||||
});
|
||||
|
||||
it('rejects old adapter-backed ingest flags at the top level and under dev', async () => {
|
||||
it('rejects old adapter-backed ingest flags at the top level and under admin', async () => {
|
||||
const rootRunIo = makeIo();
|
||||
const devRunIo = makeIo();
|
||||
const publicIngest = vi.fn(async () => 0);
|
||||
|
|
@ -997,7 +998,7 @@ describe('runKtxCli', () => {
|
|||
}),
|
||||
).resolves.toBe(1);
|
||||
await expect(
|
||||
runKtxCli(['dev', 'ingest', 'run', '--connection-id', 'warehouse', '--adapter', 'metabase'], devRunIo.io, {
|
||||
runKtxCli(['admin', 'ingest', 'run', '--connection-id', 'warehouse', '--adapter', 'metabase'], devRunIo.io, {
|
||||
publicIngest,
|
||||
}),
|
||||
).resolves.toBe(1);
|
||||
|
|
@ -1006,12 +1007,12 @@ describe('runKtxCli', () => {
|
|||
expect(devRunIo.stderr()).toMatch(/unknown command|error:/);
|
||||
});
|
||||
|
||||
it('rejects removed dev doctor and removed ingest parser cases', async () => {
|
||||
it('rejects removed admin doctor and removed ingest parser cases', async () => {
|
||||
const doctor = vi.fn(async () => 0);
|
||||
const doctorIo = makeIo();
|
||||
const ingestRunIo = makeIo();
|
||||
|
||||
await expect(runKtxCli(['dev', 'doctor', 'setup', '--json', '--no-input'], doctorIo.io, { doctor })).resolves.toBe(1);
|
||||
await expect(runKtxCli(['admin', 'doctor', 'setup', '--json', '--no-input'], doctorIo.io, { doctor })).resolves.toBe(1);
|
||||
await expect(
|
||||
runKtxCli(
|
||||
[
|
||||
|
|
@ -1803,12 +1804,12 @@ describe('runKtxCli', () => {
|
|||
expect(serveIo.stderr()).toMatch(/unknown command|error:/);
|
||||
});
|
||||
|
||||
it('prints dev help for bare dev commands', async () => {
|
||||
it('prints admin help for bare admin commands', async () => {
|
||||
const testIo = makeIo();
|
||||
|
||||
await expect(runKtxCli(['dev'], testIo.io)).resolves.toBe(0);
|
||||
await expect(runKtxCli(['admin'], testIo.io)).resolves.toBe(0);
|
||||
|
||||
expect(testIo.stdout()).toContain('Usage: ktx dev [options] [command]');
|
||||
expect(testIo.stdout()).toContain('Usage: ktx admin [options] [command]');
|
||||
expect(testIo.stdout()).toContain('Low-level project initialization');
|
||||
expect(testIo.stdout()).toContain('init');
|
||||
expect(testIo.stdout()).toContain('runtime');
|
||||
|
|
@ -1820,13 +1821,13 @@ describe('runKtxCli', () => {
|
|||
expect(testIo.stderr()).toBe('');
|
||||
});
|
||||
|
||||
it('rejects removed dev command groups without invoking execution', async () => {
|
||||
it('rejects removed admin command groups without invoking execution', async () => {
|
||||
for (const command of ['scan', 'ingest', 'mapping']) {
|
||||
const testIo = makeIo();
|
||||
const publicIngest = vi.fn().mockResolvedValue(0);
|
||||
const sl = vi.fn().mockResolvedValue(0);
|
||||
|
||||
await expect(runKtxCli(['dev', command], testIo.io, { publicIngest, sl })).resolves.toBe(1);
|
||||
await expect(runKtxCli(['admin', command], testIo.io, { publicIngest, sl })).resolves.toBe(1);
|
||||
|
||||
expect(testIo.stderr()).toMatch(/unknown command|error:/);
|
||||
expect(publicIngest).not.toHaveBeenCalled();
|
||||
|
|
@ -1834,10 +1835,10 @@ describe('runKtxCli', () => {
|
|||
}
|
||||
});
|
||||
|
||||
it('rejects removed reserved dev subcommands', async () => {
|
||||
it('rejects removed reserved admin subcommands', async () => {
|
||||
const testIo = makeIo();
|
||||
|
||||
await expect(runKtxCli(['dev', 'artifacts'], testIo.io)).resolves.toBe(1);
|
||||
await expect(runKtxCli(['admin', 'artifacts'], testIo.io)).resolves.toBe(1);
|
||||
|
||||
expect(testIo.stderr()).toMatch(/unknown command|error:/);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -118,9 +118,9 @@ function makeSpinnerEvents() {
|
|||
|
||||
describe('managedRuntimeInstallCommand', () => {
|
||||
it('prints the exact command for each managed runtime feature', () => {
|
||||
expect(managedRuntimeInstallCommand('core')).toBe('ktx dev runtime install --yes');
|
||||
expect(managedRuntimeInstallCommand('core')).toBe('ktx admin runtime install --yes');
|
||||
expect(managedRuntimeInstallCommand('local-embeddings')).toBe(
|
||||
'ktx dev runtime install --feature local-embeddings --yes',
|
||||
'ktx admin runtime install --feature local-embeddings --yes',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
@ -221,7 +221,7 @@ describe('createManagedPythonSemanticLayerComputePort', () => {
|
|||
readStatus: vi.fn(async () => missingStatus()),
|
||||
installRuntime,
|
||||
}),
|
||||
).rejects.toThrow('KTX Python runtime is required for this command. Run: ktx dev runtime install --yes');
|
||||
).rejects.toThrow('KTX Python runtime is required for this command. Run: ktx admin runtime install --yes');
|
||||
|
||||
expect(installRuntime).not.toHaveBeenCalled();
|
||||
});
|
||||
|
|
|
|||
|
|
@ -53,8 +53,8 @@ export interface ManagedPythonSemanticLayerComputeOptions extends ManagedPythonC
|
|||
|
||||
export function managedRuntimeInstallCommand(feature: KtxRuntimeFeature): string {
|
||||
return feature === 'local-embeddings'
|
||||
? 'ktx dev runtime install --feature local-embeddings --yes'
|
||||
: 'ktx dev runtime install --yes';
|
||||
? 'ktx admin runtime install --feature local-embeddings --yes'
|
||||
: 'ktx admin runtime install --yes';
|
||||
}
|
||||
|
||||
function installPrompt(feature: KtxRuntimeFeature): string {
|
||||
|
|
|
|||
|
|
@ -513,7 +513,7 @@ describe('doctorManagedPythonRuntime', () => {
|
|||
['asset', 'pass'],
|
||||
['runtime', 'fail'],
|
||||
]);
|
||||
expect(checks[2]?.fix).toBe('Run: ktx dev runtime install --yes');
|
||||
expect(checks[2]?.fix).toBe('Run: ktx admin runtime install --yes');
|
||||
});
|
||||
|
||||
it('reports uv as a hard prerequisite when uv is missing', async () => {
|
||||
|
|
@ -534,7 +534,7 @@ describe('doctorManagedPythonRuntime', () => {
|
|||
label: 'uv',
|
||||
status: 'fail',
|
||||
detail: MISSING_UV_RUNTIME_INSTALL_MESSAGE,
|
||||
fix: 'Install uv, make sure it is on PATH, and run: ktx dev runtime install --yes',
|
||||
fix: 'Install uv, make sure it is on PATH, and run: ktx admin runtime install --yes',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -122,7 +122,7 @@ export interface ManagedPythonRuntimeDoctorCheck {
|
|||
}
|
||||
|
||||
export const MISSING_UV_RUNTIME_INSTALL_MESSAGE =
|
||||
'uv is required to install the KTX Python runtime. KTX does not download uv automatically. Install uv, make sure it is on PATH, and retry: ktx dev runtime install --yes';
|
||||
'uv is required to install the KTX Python runtime. KTX does not download uv automatically. Install uv, make sure it is on PATH, and retry: ktx admin runtime install --yes';
|
||||
|
||||
function defaultAssetDir(): string {
|
||||
return fileURLToPath(new URL('../assets/python/', import.meta.url));
|
||||
|
|
@ -471,7 +471,7 @@ export async function doctorManagedPythonRuntime(
|
|||
id: 'uv',
|
||||
label: 'uv',
|
||||
detail: error instanceof Error ? error.message : String(error),
|
||||
fix: 'Install uv, make sure it is on PATH, and run: ktx dev runtime install --yes',
|
||||
fix: 'Install uv, make sure it is on PATH, and run: ktx admin runtime install --yes',
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
|
@ -496,7 +496,7 @@ export async function doctorManagedPythonRuntime(
|
|||
id: 'runtime',
|
||||
label: 'Managed Python runtime',
|
||||
detail: status.detail,
|
||||
...(status.kind === 'ready' ? {} : { fix: 'Run: ktx dev runtime install --yes' }),
|
||||
...(status.kind === 'ready' ? {} : { fix: 'Run: ktx admin runtime install --yes' }),
|
||||
}),
|
||||
);
|
||||
return checks;
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ describe('renderKtxCommandTree', () => {
|
|||
.filter((line) => /^ {2}[├└]── \S/.test(line))
|
||||
.map((line) => line.replace(/^ {2}[├└]── /, '').trim().split(' ')[0]);
|
||||
|
||||
for (const expected of ['setup', 'connection', 'ingest', 'sl', 'mcp', 'dev']) {
|
||||
for (const expected of ['setup', 'connection', 'ingest', 'sl', 'mcp', 'admin']) {
|
||||
expect(topLevel).toContain(expected);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -291,7 +291,7 @@ describe('runKtxRuntime', () => {
|
|||
label: 'Managed Python runtime',
|
||||
status: 'fail',
|
||||
detail: 'No runtime manifest at /runtime/0.2.0/manifest.json',
|
||||
fix: 'Run: ktx dev runtime install --yes',
|
||||
fix: 'Run: ktx admin runtime install --yes',
|
||||
},
|
||||
]),
|
||||
};
|
||||
|
|
|
|||
|
|
@ -368,8 +368,8 @@ describe('runKtxScan', () => {
|
|||
expect(io.stdout()).toContain('Report: raw-sources/warehouse/live-database/sync-1/scan-report.json');
|
||||
expect(io.stdout()).toContain('Next:\n');
|
||||
expect(io.stdout()).toContain('ktx status --project-dir ');
|
||||
expect(io.stdout()).not.toContain('ktx dev scan status');
|
||||
expect(io.stdout()).not.toContain('ktx dev scan report');
|
||||
expect(io.stdout()).not.toContain('ktx admin scan status');
|
||||
expect(io.stdout()).not.toContain('ktx admin scan report');
|
||||
expect(io.stdout()).not.toContain('\u001b[');
|
||||
expect(io.stdout()).not.toContain('✓');
|
||||
expect(io.stdout()).not.toContain('+1');
|
||||
|
|
|
|||
|
|
@ -286,7 +286,7 @@ describe('setup embeddings step', () => {
|
|||
const io = makeIo();
|
||||
const ensureLocalEmbeddings = vi.fn(async () => {
|
||||
throw new Error(
|
||||
'KTX Python runtime is required for this command. Run: ktx dev runtime install --feature local-embeddings --yes',
|
||||
'KTX Python runtime is required for this command. Run: ktx admin runtime install --feature local-embeddings --yes',
|
||||
);
|
||||
});
|
||||
|
||||
|
|
@ -304,7 +304,7 @@ describe('setup embeddings step', () => {
|
|||
|
||||
expect(result.status).toBe('failed');
|
||||
expect(io.stderr()).toContain(
|
||||
'KTX Python runtime is required for this command. Run: ktx dev runtime install --feature local-embeddings --yes',
|
||||
'KTX Python runtime is required for this command. Run: ktx admin runtime install --feature local-embeddings --yes',
|
||||
);
|
||||
});
|
||||
|
||||
|
|
@ -331,7 +331,7 @@ describe('setup embeddings step', () => {
|
|||
expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:');
|
||||
expect(config.ingest.embeddings.backend).toBe('none');
|
||||
expect(io.stderr()).toContain('Local embedding health check failed: 401 invalid api key [redacted]');
|
||||
expect(io.stderr()).toContain('Prepare the runtime with: ktx dev runtime start --feature local-embeddings');
|
||||
expect(io.stderr()).toContain('Prepare the runtime with: ktx admin runtime start --feature local-embeddings');
|
||||
expect(io.stderr()).not.toContain('skip for now');
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -307,7 +307,7 @@ function localEmbeddingSetupMessage(message: string, stderrTail: string[] = []):
|
|||
const lines = [
|
||||
`Local embedding health check failed: ${message}`,
|
||||
'Local embeddings use the KTX-managed Python runtime.',
|
||||
'Prepare the runtime with: ktx dev runtime start --feature local-embeddings',
|
||||
'Prepare the runtime with: ktx admin runtime start --feature local-embeddings',
|
||||
'Use --yes with setup to install and start the runtime without prompting.',
|
||||
'The first run may download Python packages and the all-MiniLM-L6-v2 model.',
|
||||
];
|
||||
|
|
|
|||
|
|
@ -71,7 +71,7 @@ describe('runKtxSetupRuntimeStep', () => {
|
|||
it('fails fast when required runtime features cannot be installed in no-input mode', async () => {
|
||||
const io = makeIo();
|
||||
const ensureRuntime = vi.fn(async () => {
|
||||
throw new Error('KTX Python runtime is required for this command. Run: ktx dev runtime install --yes');
|
||||
throw new Error('KTX Python runtime is required for this command. Run: ktx admin runtime install --yes');
|
||||
});
|
||||
|
||||
await expect(
|
||||
|
|
@ -94,7 +94,7 @@ describe('runKtxSetupRuntimeStep', () => {
|
|||
|
||||
expect(ensureRuntime).toHaveBeenCalledWith(expect.objectContaining({ installPolicy: 'never' }));
|
||||
expect((await readKtxSetupState(tempDir)).completed_steps).not.toContain('runtime');
|
||||
expect(io.stderr()).toContain('ktx dev runtime install --yes');
|
||||
expect(io.stderr()).toContain('ktx admin runtime install --yes');
|
||||
});
|
||||
|
||||
it('starts the managed local embeddings daemon for configured sentence-transformers embeddings', async () => {
|
||||
|
|
|
|||
|
|
@ -144,6 +144,11 @@ describe('standalone built ktx CLI smoke', () => {
|
|||
expectSetupStderr(init);
|
||||
expect(init.stdout).toContain(`Project: ${projectDir}`);
|
||||
|
||||
const reindex = await runBuiltCli(['--project-dir', projectDir, 'admin', 'reindex', '--output', 'plain']);
|
||||
expect(reindex.code).toBe(0);
|
||||
expect(reindex.stdout).toContain('reindex\t');
|
||||
expect(reindex.stderr).toContain('wiki/global');
|
||||
|
||||
const run = await runBuiltCli([
|
||||
'ingest',
|
||||
'run',
|
||||
|
|
|
|||
|
|
@ -50,6 +50,11 @@
|
|||
"import": "./dist/ingest/metabase-mapping.js",
|
||||
"default": "./dist/ingest/metabase-mapping.js"
|
||||
},
|
||||
"./index-sync": {
|
||||
"types": "./dist/index-sync/index.d.ts",
|
||||
"import": "./dist/index-sync/index.js",
|
||||
"default": "./dist/index-sync/index.js"
|
||||
},
|
||||
"./scan": {
|
||||
"types": "./dist/scan/index.d.ts",
|
||||
"import": "./dist/scan/index.js",
|
||||
|
|
|
|||
2
packages/context/src/index-sync/index.ts
Normal file
2
packages/context/src/index-sync/index.ts
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
export type { ReindexOptions, ReindexScopeResult, ReindexSummary, ReindexWorkResult } from './types.js';
|
||||
export { discoverReindexScopes, reindexLocalIndexes } from './reindex.js';
|
||||
196
packages/context/src/index-sync/reindex.test.ts
Normal file
196
packages/context/src/index-sync/reindex.test.ts
Normal file
|
|
@ -0,0 +1,196 @@
|
|||
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
import type { KtxEmbeddingPort } from '../core/index.js';
|
||||
import { initKtxProject, loadKtxProject, type KtxLocalProject } from '../project/index.js';
|
||||
import { SqliteKnowledgeIndex } from '../wiki/sqlite-knowledge-index.js';
|
||||
import { reindexLocalIndexes } from './reindex.js';
|
||||
|
||||
class FakeEmbeddingPort implements KtxEmbeddingPort {
|
||||
readonly maxBatchSize = 8;
|
||||
|
||||
async computeEmbedding(text: string): Promise<number[]> {
|
||||
return [text.length, 1];
|
||||
}
|
||||
|
||||
async computeEmbeddingsBulk(texts: string[]): Promise<number[][]> {
|
||||
return texts.map((text) => [text.length, 1]);
|
||||
}
|
||||
}
|
||||
|
||||
async function createProject(tempDir: string): Promise<KtxLocalProject> {
|
||||
await initKtxProject({ projectDir: tempDir, force: true });
|
||||
return loadKtxProject({ projectDir: tempDir });
|
||||
}
|
||||
|
||||
describe('reindexLocalIndexes', () => {
|
||||
let tempDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'ktx-reindex-'));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('returns an empty summary when no wiki or semantic-layer directories exist', async () => {
|
||||
const project = await createProject(tempDir);
|
||||
await rm(join(project.projectDir, 'wiki'), { recursive: true, force: true });
|
||||
await rm(join(project.projectDir, 'semantic-layer'), { recursive: true, force: true });
|
||||
|
||||
await expect(reindexLocalIndexes(project, { force: false, embeddingService: null })).resolves.toMatchObject({
|
||||
scopes: [],
|
||||
totals: { scanned: 0, updated: 0, deleted: 0, embeddingsRecomputed: 0, embeddingsFailed: 0 },
|
||||
force: false,
|
||||
embeddingsAvailable: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('discovers empty directories as zero-row scopes', async () => {
|
||||
const project = await createProject(tempDir);
|
||||
await mkdir(join(project.projectDir, 'wiki/user/local'), { recursive: true });
|
||||
await mkdir(join(project.projectDir, 'semantic-layer/warehouse'), { recursive: true });
|
||||
|
||||
const summary = await reindexLocalIndexes(project, { force: false, embeddingService: null });
|
||||
|
||||
expect(summary.scopes.map((scope) => scope.label)).toEqual(['global', 'user/local', 'warehouse']);
|
||||
expect(summary.totals.scanned).toBe(0);
|
||||
});
|
||||
|
||||
it('indexes mixed wiki and SL sources and reports totals', async () => {
|
||||
const project = await createProject(tempDir);
|
||||
await writeFile(
|
||||
join(project.projectDir, 'wiki/global/revenue.md'),
|
||||
'---\nsummary: Revenue\nusage_mode: auto\n---\n\nPaid orders.\n',
|
||||
'utf-8',
|
||||
);
|
||||
await mkdir(join(project.projectDir, 'semantic-layer/warehouse'), { recursive: true });
|
||||
await writeFile(
|
||||
join(project.projectDir, 'semantic-layer/warehouse/orders.yaml'),
|
||||
'name: orders\ntable: public.orders\ngrain: [id]\ncolumns:\n - name: id\n type: number\njoins: []\nmeasures: []\n',
|
||||
'utf-8',
|
||||
);
|
||||
|
||||
const summary = await reindexLocalIndexes(project, {
|
||||
force: false,
|
||||
embeddingService: new FakeEmbeddingPort(),
|
||||
});
|
||||
|
||||
expect(summary.scopes).toHaveLength(2);
|
||||
expect(summary.totals).toMatchObject({ scanned: 2, updated: 2, deleted: 0, embeddingsRecomputed: 2 });
|
||||
expect(summary.embeddingsAvailable).toBe(true);
|
||||
});
|
||||
|
||||
it('does not report unchanged lexical-only rows as updated on repeated runs', async () => {
|
||||
const project = await createProject(tempDir);
|
||||
await writeFile(
|
||||
join(project.projectDir, 'wiki/global/revenue.md'),
|
||||
'---\nsummary: Revenue\nusage_mode: auto\n---\n\nPaid orders.\n',
|
||||
'utf-8',
|
||||
);
|
||||
await mkdir(join(project.projectDir, 'semantic-layer/warehouse'), { recursive: true });
|
||||
await writeFile(
|
||||
join(project.projectDir, 'semantic-layer/warehouse/orders.yaml'),
|
||||
'name: orders\ntable: public.orders\ngrain: [id]\ncolumns:\n - name: id\n type: number\njoins: []\nmeasures: []\n',
|
||||
'utf-8',
|
||||
);
|
||||
|
||||
const first = await reindexLocalIndexes(project, { force: false, embeddingService: null });
|
||||
expect(first.totals).toMatchObject({
|
||||
scanned: 2,
|
||||
updated: 2,
|
||||
deleted: 0,
|
||||
embeddingsRecomputed: 0,
|
||||
embeddingsFailed: 0,
|
||||
});
|
||||
|
||||
const second = await reindexLocalIndexes(project, { force: false, embeddingService: null });
|
||||
|
||||
expect(second.totals).toMatchObject({
|
||||
scanned: 2,
|
||||
updated: 0,
|
||||
deleted: 0,
|
||||
embeddingsRecomputed: 0,
|
||||
embeddingsFailed: 0,
|
||||
});
|
||||
expect(second.scopes.map((scope) => [scope.label, scope.updated])).toEqual([
|
||||
['global', 0],
|
||||
['warehouse', 0],
|
||||
]);
|
||||
});
|
||||
|
||||
it('force clears stale rows before rebuilding each discovered scope', async () => {
|
||||
const project = await createProject(tempDir);
|
||||
const wikiIndex = new SqliteKnowledgeIndex({ dbPath: join(project.projectDir, '.ktx/db.sqlite') });
|
||||
wikiIndex.sync([
|
||||
{
|
||||
path: 'wiki/global/stale.md',
|
||||
key: 'stale',
|
||||
scope: 'GLOBAL',
|
||||
scopeId: null,
|
||||
summary: 'Stale',
|
||||
content: 'Stale content',
|
||||
tags: [],
|
||||
embedding: [1, 0],
|
||||
},
|
||||
]);
|
||||
await writeFile(
|
||||
join(project.projectDir, 'wiki/global/revenue.md'),
|
||||
'---\nsummary: Revenue\nusage_mode: auto\n---\n\nPaid orders.\n',
|
||||
'utf-8',
|
||||
);
|
||||
|
||||
const summary = await reindexLocalIndexes(project, {
|
||||
force: true,
|
||||
embeddingService: new FakeEmbeddingPort(),
|
||||
});
|
||||
|
||||
expect(summary.force).toBe(true);
|
||||
expect(summary.totals).toMatchObject({ scanned: 1, updated: 1, deleted: 0 });
|
||||
expect(wikiIndex.search('Stale', 10)).toEqual([]);
|
||||
});
|
||||
|
||||
it('captures a per-scope error and continues other scopes', async () => {
|
||||
const project = await createProject(tempDir);
|
||||
await writeFile(
|
||||
join(project.projectDir, 'wiki/global/revenue.md'),
|
||||
'---\nsummary: Revenue\nusage_mode: auto\n---\n\nPaid orders.\n',
|
||||
'utf-8',
|
||||
);
|
||||
await mkdir(join(project.projectDir, 'semantic-layer/warehouse'), { recursive: true });
|
||||
await writeFile(join(project.projectDir, 'semantic-layer/warehouse/broken.yaml'), 'not: [valid', 'utf-8');
|
||||
|
||||
const summary = await reindexLocalIndexes(project, { force: false, embeddingService: null });
|
||||
|
||||
expect(summary.scopes.find((scope) => scope.label === 'global')?.error).toBeUndefined();
|
||||
expect(summary.scopes.find((scope) => scope.label === 'warehouse')?.error).toContain('YAML');
|
||||
});
|
||||
|
||||
it('marks a scope errored when configured embeddings fail', async () => {
|
||||
const project = await createProject(tempDir);
|
||||
await writeFile(
|
||||
join(project.projectDir, 'wiki/global/revenue.md'),
|
||||
'---\nsummary: Revenue\nusage_mode: auto\n---\n\nPaid orders.\n',
|
||||
'utf-8',
|
||||
);
|
||||
const embeddingService: KtxEmbeddingPort = {
|
||||
maxBatchSize: 8,
|
||||
async computeEmbedding() {
|
||||
throw new Error('embedding provider unavailable');
|
||||
},
|
||||
async computeEmbeddingsBulk() {
|
||||
throw new Error('embedding provider unavailable');
|
||||
},
|
||||
};
|
||||
|
||||
const summary = await reindexLocalIndexes(project, { force: false, embeddingService });
|
||||
|
||||
expect(summary.scopes[0]).toMatchObject({
|
||||
label: 'global',
|
||||
embeddingsFailed: 1,
|
||||
error: '1 embedding recomputation failed',
|
||||
});
|
||||
});
|
||||
});
|
||||
162
packages/context/src/index-sync/reindex.ts
Normal file
162
packages/context/src/index-sync/reindex.ts
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
import { readdir, stat } from 'node:fs/promises';
|
||||
import { join, relative } from 'node:path';
|
||||
import { ktxLocalStateDbPath, type KtxLocalProject } from '../project/index.js';
|
||||
import { loadLocalSlSourceRecords, SlSearchService, SqliteSlSourcesIndex } from '../sl/index.js';
|
||||
import { KnowledgeWikiService, SqliteKnowledgeIndex } from '../wiki/index.js';
|
||||
import type { ReindexOptions, ReindexScopeResult, ReindexSummary, ReindexWorkResult } from './types.js';
|
||||
|
||||
type DiscoveredScope =
|
||||
| { kind: 'wiki'; scope: 'GLOBAL'; scopeId: null; label: 'global' }
|
||||
| { kind: 'wiki'; scope: 'USER'; scopeId: string; label: `user/${string}` }
|
||||
| { kind: 'sl'; connectionId: string; label: string };
|
||||
|
||||
const ZERO: ReindexWorkResult = {
|
||||
scanned: 0,
|
||||
updated: 0,
|
||||
deleted: 0,
|
||||
embeddingsRecomputed: 0,
|
||||
embeddingsFailed: 0,
|
||||
};
|
||||
|
||||
async function directoryExists(path: string): Promise<boolean> {
|
||||
try {
|
||||
return (await stat(path)).isDirectory();
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function childDirectories(path: string): Promise<string[]> {
|
||||
try {
|
||||
const entries = await readdir(path, { withFileTypes: true });
|
||||
return entries
|
||||
.filter((entry) => entry.isDirectory())
|
||||
.map((entry) => entry.name)
|
||||
.sort((left, right) => left.localeCompare(right));
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
|
||||
return [];
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export async function discoverReindexScopes(project: KtxLocalProject): Promise<DiscoveredScope[]> {
|
||||
const scopes: DiscoveredScope[] = [];
|
||||
if (await directoryExists(join(project.projectDir, 'wiki/global'))) {
|
||||
scopes.push({ kind: 'wiki', scope: 'GLOBAL', scopeId: null, label: 'global' });
|
||||
}
|
||||
for (const userId of await childDirectories(join(project.projectDir, 'wiki/user'))) {
|
||||
scopes.push({ kind: 'wiki', scope: 'USER', scopeId: userId, label: `user/${userId}` });
|
||||
}
|
||||
for (const connectionId of await childDirectories(join(project.projectDir, 'semantic-layer'))) {
|
||||
if (connectionId !== '_schema') {
|
||||
scopes.push({ kind: 'sl', connectionId, label: connectionId });
|
||||
}
|
||||
}
|
||||
return scopes;
|
||||
}
|
||||
|
||||
function errorMessage(error: unknown): string {
|
||||
if (!(error instanceof Error)) {
|
||||
return String(error);
|
||||
}
|
||||
return error.name && error.name !== 'Error' ? `${error.name}: ${error.message}` : error.message;
|
||||
}
|
||||
|
||||
function addTotals(left: ReindexWorkResult, right: ReindexWorkResult): ReindexWorkResult {
|
||||
return {
|
||||
scanned: left.scanned + right.scanned,
|
||||
updated: left.updated + right.updated,
|
||||
deleted: left.deleted + right.deleted,
|
||||
embeddingsRecomputed: left.embeddingsRecomputed + right.embeddingsRecomputed,
|
||||
embeddingsFailed: left.embeddingsFailed + right.embeddingsFailed,
|
||||
};
|
||||
}
|
||||
|
||||
function durationSince(startedAt: bigint): number {
|
||||
return Number((process.hrtime.bigint() - startedAt) / 1_000_000n);
|
||||
}
|
||||
|
||||
function embeddingFailureError(work: ReindexWorkResult): string | undefined {
|
||||
if (work.embeddingsFailed === 0) {
|
||||
return undefined;
|
||||
}
|
||||
return `${work.embeddingsFailed} embedding recomputation${work.embeddingsFailed === 1 ? '' : 's'} failed`;
|
||||
}
|
||||
|
||||
export async function reindexLocalIndexes(
|
||||
project: KtxLocalProject,
|
||||
options: ReindexOptions,
|
||||
): Promise<ReindexSummary> {
|
||||
const startedAt = process.hrtime.bigint();
|
||||
const dbPath = ktxLocalStateDbPath(project);
|
||||
const scopes = await discoverReindexScopes(project);
|
||||
const wikiIndex = new SqliteKnowledgeIndex({ dbPath });
|
||||
const slIndex = new SqliteSlSourcesIndex({ dbPath });
|
||||
const wikiService = new KnowledgeWikiService(project.fileStore, options.embeddingService, wikiIndex, project.git);
|
||||
const slService = new SlSearchService(options.embeddingService, slIndex);
|
||||
const results: ReindexScopeResult[] = [];
|
||||
|
||||
for (const scope of scopes) {
|
||||
const scopeStartedAt = process.hrtime.bigint();
|
||||
try {
|
||||
let work: ReindexWorkResult;
|
||||
if (scope.kind === 'wiki') {
|
||||
if (options.force) {
|
||||
wikiIndex.clear(scope.scope, scope.scopeId);
|
||||
}
|
||||
work = await wikiService.syncIndex(scope.scope, scope.scopeId);
|
||||
results.push({
|
||||
kind: 'wiki',
|
||||
label: scope.label,
|
||||
scope: scope.scope === 'GLOBAL' ? 'global' : 'user',
|
||||
scopeId: scope.scopeId,
|
||||
...work,
|
||||
...(options.force ? { deleted: 0 } : {}),
|
||||
...(options.embeddingService && work.embeddingsFailed > 0 ? { error: embeddingFailureError(work) } : {}),
|
||||
durationMs: durationSince(scopeStartedAt),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
if (options.force) {
|
||||
await slIndex.clear(scope.connectionId);
|
||||
}
|
||||
const records = await loadLocalSlSourceRecords(project, { connectionId: scope.connectionId });
|
||||
work = await slService.indexSources(
|
||||
scope.connectionId,
|
||||
records.map((record) => record.source),
|
||||
);
|
||||
results.push({
|
||||
kind: 'sl',
|
||||
label: scope.label,
|
||||
connectionId: scope.connectionId,
|
||||
...work,
|
||||
...(options.force ? { deleted: 0 } : {}),
|
||||
...(options.embeddingService && work.embeddingsFailed > 0 ? { error: embeddingFailureError(work) } : {}),
|
||||
durationMs: durationSince(scopeStartedAt),
|
||||
});
|
||||
} catch (error) {
|
||||
results.push({
|
||||
kind: scope.kind,
|
||||
label: scope.label,
|
||||
...(scope.kind === 'wiki'
|
||||
? { scope: scope.scope === 'GLOBAL' ? 'global' : 'user', scopeId: scope.scopeId }
|
||||
: { connectionId: scope.connectionId }),
|
||||
...ZERO,
|
||||
durationMs: durationSince(scopeStartedAt),
|
||||
error: errorMessage(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
scopes: results,
|
||||
totals: results.reduce(addTotals, ZERO),
|
||||
dbPath: relative(project.projectDir, dbPath) || dbPath,
|
||||
force: options.force,
|
||||
embeddingsAvailable: options.embeddingService !== null,
|
||||
durationMs: durationSince(startedAt),
|
||||
};
|
||||
}
|
||||
33
packages/context/src/index-sync/types.ts
Normal file
33
packages/context/src/index-sync/types.ts
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
import type { KtxEmbeddingPort } from '../core/index.js';
|
||||
|
||||
export interface ReindexOptions {
|
||||
force: boolean;
|
||||
embeddingService: KtxEmbeddingPort | null;
|
||||
}
|
||||
|
||||
export interface ReindexWorkResult {
|
||||
scanned: number;
|
||||
updated: number;
|
||||
deleted: number;
|
||||
embeddingsRecomputed: number;
|
||||
embeddingsFailed: number;
|
||||
}
|
||||
|
||||
export interface ReindexScopeResult extends ReindexWorkResult {
|
||||
kind: 'wiki' | 'sl';
|
||||
label: string;
|
||||
scope?: 'global' | 'user';
|
||||
scopeId?: string | null;
|
||||
connectionId?: string;
|
||||
durationMs: number;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface ReindexSummary {
|
||||
scopes: ReindexScopeResult[];
|
||||
totals: ReindexWorkResult;
|
||||
dbPath: string;
|
||||
force: boolean;
|
||||
embeddingsAvailable: boolean;
|
||||
durationMs: number;
|
||||
}
|
||||
|
|
@ -12,6 +12,7 @@ export * from './agent/index.js';
|
|||
export * from './core/index.js';
|
||||
export * from './daemon/index.js';
|
||||
export * from './ingest/index.js';
|
||||
export * from './index-sync/index.js';
|
||||
export * from './llm/index.js';
|
||||
export type {
|
||||
CaptureSession,
|
||||
|
|
|
|||
|
|
@ -380,16 +380,19 @@ class LocalKnowledgeIndex implements KnowledgeIndexPort {
|
|||
return result;
|
||||
}
|
||||
|
||||
async deleteStale(): Promise<void> {
|
||||
async deleteStale(): Promise<number> {
|
||||
await this.syncAllPagesFromDisk();
|
||||
return 0;
|
||||
}
|
||||
|
||||
async deleteByScope(): Promise<void> {
|
||||
async deleteByScope(): Promise<number> {
|
||||
await this.syncAllPagesFromDisk();
|
||||
return 0;
|
||||
}
|
||||
|
||||
async deleteByKey(): Promise<void> {
|
||||
async deleteByKey(): Promise<number> {
|
||||
await this.syncAllPagesFromDisk();
|
||||
return 0;
|
||||
}
|
||||
|
||||
async findPageByKey(scope: string, scopeId: string | null, pageKey: string) {
|
||||
|
|
|
|||
|
|
@ -205,11 +205,17 @@ class LocalKnowledgeIndex implements KnowledgeIndexPort {
|
|||
return new Map();
|
||||
}
|
||||
|
||||
async deleteStale(): Promise<void> {}
|
||||
async deleteStale(): Promise<number> {
|
||||
return 0;
|
||||
}
|
||||
|
||||
async deleteByScope(): Promise<void> {}
|
||||
async deleteByScope(): Promise<number> {
|
||||
return 0;
|
||||
}
|
||||
|
||||
async deleteByKey(): Promise<void> {}
|
||||
async deleteByKey(): Promise<number> {
|
||||
return 0;
|
||||
}
|
||||
|
||||
async findPageByKey(scope: string, scopeId: string | null, pageKey: string) {
|
||||
const path = this.pagePath(scope, scopeId, pageKey);
|
||||
|
|
|
|||
|
|
@ -40,9 +40,9 @@ export interface SlSourcesIndexPort {
|
|||
sources: Array<{ sourceName: string; searchText: string; embedding: number[] | null; contentHash?: string | null }>,
|
||||
): Promise<void>;
|
||||
getExistingSearchTexts(connectionId: string): Promise<Map<string, { searchText: string; hasEmbedding: boolean }>>;
|
||||
deleteStale(connectionId: string, keepNames: string[]): Promise<void>;
|
||||
deleteByConnection(connectionId: string): Promise<void>;
|
||||
deleteByConnectionAndName(connectionId: string, sourceName: string): Promise<void>;
|
||||
deleteStale(connectionId: string, keepNames: string[]): Promise<number>;
|
||||
deleteByConnection(connectionId: string): Promise<number>;
|
||||
deleteByConnectionAndName(connectionId: string, sourceName: string): Promise<number>;
|
||||
search(
|
||||
connectionId: string,
|
||||
queryEmbedding: number[] | null,
|
||||
|
|
|
|||
|
|
@ -223,4 +223,73 @@ describe('SlSearchService', () => {
|
|||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('indexSources reports stats and supports lexical-only indexing', async () => {
|
||||
const repository = {
|
||||
upsertSources: vi.fn().mockResolvedValue(undefined),
|
||||
getExistingSearchTexts: vi.fn().mockResolvedValue(
|
||||
new Map([
|
||||
['old_source', { searchText: 'old source', hasEmbedding: true }],
|
||||
]),
|
||||
),
|
||||
deleteStale: vi.fn().mockResolvedValue(1),
|
||||
deleteByConnection: vi.fn().mockResolvedValue(0),
|
||||
deleteByConnectionAndName: vi.fn(),
|
||||
search: vi.fn(),
|
||||
};
|
||||
const service = new SlSearchService(null, repository);
|
||||
const source: SemanticLayerSource = {
|
||||
name: 'orders',
|
||||
table: 'public.orders',
|
||||
grain: ['id'],
|
||||
columns: [{ name: 'id', type: 'number' }],
|
||||
joins: [],
|
||||
measures: [],
|
||||
};
|
||||
|
||||
await expect(service.indexSources('warehouse', [source])).resolves.toEqual({
|
||||
scanned: 1,
|
||||
updated: 1,
|
||||
deleted: 1,
|
||||
embeddingsRecomputed: 0,
|
||||
embeddingsFailed: 0,
|
||||
});
|
||||
expect(repository.upsertSources).toHaveBeenCalledWith('warehouse', [
|
||||
expect.objectContaining({ sourceName: 'orders', embedding: null }),
|
||||
]);
|
||||
});
|
||||
|
||||
it('does not update unchanged lexical-only SL rows on repeated sync', async () => {
|
||||
const repository = {
|
||||
upsertSources: vi.fn().mockResolvedValue(undefined),
|
||||
getExistingSearchTexts: vi.fn().mockResolvedValue(
|
||||
new Map([
|
||||
['orders', { searchText: 'orders. table: public.orders. id (number)', hasEmbedding: false }],
|
||||
]),
|
||||
),
|
||||
deleteStale: vi.fn().mockResolvedValue(0),
|
||||
deleteByConnection: vi.fn().mockResolvedValue(0),
|
||||
deleteByConnectionAndName: vi.fn(),
|
||||
search: vi.fn(),
|
||||
};
|
||||
const service = new SlSearchService(null, repository);
|
||||
const source: SemanticLayerSource = {
|
||||
name: 'orders',
|
||||
table: 'public.orders',
|
||||
grain: ['id'],
|
||||
columns: [{ name: 'id', type: 'number' }],
|
||||
joins: [],
|
||||
measures: [],
|
||||
};
|
||||
|
||||
await expect(service.indexSources('warehouse', [source])).resolves.toEqual({
|
||||
scanned: 1,
|
||||
updated: 0,
|
||||
deleted: 0,
|
||||
embeddingsRecomputed: 0,
|
||||
embeddingsFailed: 0,
|
||||
});
|
||||
expect(repository.upsertSources).toHaveBeenCalledWith('warehouse', []);
|
||||
expect(repository.deleteStale).toHaveBeenCalledWith('warehouse', ['orders']);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import type { KtxEmbeddingPort, KtxLogger } from '../core/index.js';
|
||||
import { noopLogger } from '../core/index.js';
|
||||
import type { ReindexWorkResult } from '../index-sync/types.js';
|
||||
import { DEFAULT_PRIORITY, resolveDescription } from './descriptions.js';
|
||||
import { normalizeSemanticLayerDescriptions } from './description-normalization.js';
|
||||
import type { SlSourcesIndexPort } from './ports.js';
|
||||
|
|
@ -94,73 +95,71 @@ export function buildSemanticLayerSourceSearchText(
|
|||
|
||||
export class SlSearchService {
|
||||
constructor(
|
||||
private readonly embeddingService: KtxEmbeddingPort,
|
||||
private readonly embeddingService: KtxEmbeddingPort | null,
|
||||
private readonly slSourcesRepository: SlSourcesIndexPort,
|
||||
private readonly logger: KtxLogger = noopLogger,
|
||||
) {}
|
||||
|
||||
async indexSources(connectionId: string, sources: SemanticLayerSource[]): Promise<void> {
|
||||
async indexSources(connectionId: string, sources: SemanticLayerSource[]): Promise<ReindexWorkResult> {
|
||||
const existing = await this.slSourcesRepository.getExistingSearchTexts(connectionId);
|
||||
if (sources.length === 0) {
|
||||
await this.slSourcesRepository.deleteByConnection(connectionId);
|
||||
return;
|
||||
const deleted = await this.slSourcesRepository.deleteByConnection(connectionId);
|
||||
return { scanned: 0, updated: 0, deleted, embeddingsRecomputed: 0, embeddingsFailed: 0 };
|
||||
}
|
||||
|
||||
// Detect which sources actually changed by comparing search_text
|
||||
const existing = await this.slSourcesRepository.getExistingSearchTexts(connectionId);
|
||||
const searchTexts = sources.map((s) => this.buildSearchText(s));
|
||||
|
||||
const embeddingService = this.embeddingService;
|
||||
const changedIndices: number[] = [];
|
||||
for (let i = 0; i < sources.length; i++) {
|
||||
const prev = existing.get(sources[i].name);
|
||||
if (!prev || prev.searchText !== searchTexts[i] || !prev.hasEmbedding) {
|
||||
for (let i = 0; i < sources.length; i += 1) {
|
||||
const previous = existing.get(sources[i]!.name);
|
||||
if (
|
||||
!previous ||
|
||||
previous.searchText !== searchTexts[i] ||
|
||||
(embeddingService !== null && !previous.hasEmbedding)
|
||||
) {
|
||||
changedIndices.push(i);
|
||||
}
|
||||
}
|
||||
|
||||
if (changedIndices.length === 0) {
|
||||
// Still clean up stale sources even if nothing changed
|
||||
const keepNames = sources.map((s) => s.name);
|
||||
await this.slSourcesRepository.deleteStale(connectionId, keepNames);
|
||||
this.logger.log(`SL sources for connection ${connectionId}: all ${sources.length} up to date, 0 reindexed`);
|
||||
return;
|
||||
}
|
||||
let changedEmbeddings: (number[] | null)[] = changedIndices.map(() => null);
|
||||
let embeddingsRecomputed = 0;
|
||||
let embeddingsFailed = 0;
|
||||
|
||||
// Compute embeddings only for changed sources
|
||||
const changedTexts = changedIndices.map((i) => searchTexts[i]);
|
||||
let changedEmbeddings: (number[] | null)[];
|
||||
try {
|
||||
const batchSize = this.embeddingService.maxBatchSize;
|
||||
const allEmbeddings: number[][] = [];
|
||||
for (let i = 0; i < changedTexts.length; i += batchSize) {
|
||||
const batch = changedTexts.slice(i, i + batchSize);
|
||||
const batchEmbeddings = await this.embeddingService.computeEmbeddingsBulk(batch);
|
||||
allEmbeddings.push(...batchEmbeddings);
|
||||
if (embeddingService && changedIndices.length > 0) {
|
||||
try {
|
||||
const changedTexts = changedIndices.map((index) => searchTexts[index]!);
|
||||
const allEmbeddings: number[][] = [];
|
||||
for (let i = 0; i < changedTexts.length; i += embeddingService.maxBatchSize) {
|
||||
const batch = changedTexts.slice(i, i + embeddingService.maxBatchSize);
|
||||
allEmbeddings.push(...(await embeddingService.computeEmbeddingsBulk(batch)));
|
||||
}
|
||||
changedEmbeddings = allEmbeddings;
|
||||
embeddingsRecomputed = allEmbeddings.length;
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
`Failed to compute SL source embeddings: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
embeddingsFailed = changedIndices.length;
|
||||
}
|
||||
changedEmbeddings = allEmbeddings;
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
`Failed to compute SL source embeddings: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
changedEmbeddings = changedIndices.map(() => null);
|
||||
}
|
||||
|
||||
const rows = changedIndices.map((srcIdx, i) => {
|
||||
return {
|
||||
sourceName: sources[srcIdx].name,
|
||||
searchText: searchTexts[srcIdx],
|
||||
embedding: changedEmbeddings[i],
|
||||
};
|
||||
});
|
||||
|
||||
const rows = changedIndices.map((sourceIndex, embeddingIndex) => ({
|
||||
sourceName: sources[sourceIndex]!.name,
|
||||
searchText: searchTexts[sourceIndex]!,
|
||||
embedding: changedEmbeddings[embeddingIndex] ?? null,
|
||||
}));
|
||||
await this.slSourcesRepository.upsertSources(connectionId, rows);
|
||||
|
||||
// Remove sources that no longer exist in YAML
|
||||
const keepNames = sources.map((s) => s.name);
|
||||
await this.slSourcesRepository.deleteStale(connectionId, keepNames);
|
||||
|
||||
this.logger.log(
|
||||
`SL sources for connection ${connectionId}: ${changedIndices.length}/${sources.length} reindexed, ${sources.length - changedIndices.length} unchanged`,
|
||||
);
|
||||
const keepNames = sources.map((source) => source.name);
|
||||
const deleted = await this.slSourcesRepository.deleteStale(connectionId, keepNames);
|
||||
return {
|
||||
scanned: sources.length,
|
||||
updated: changedIndices.length,
|
||||
deleted,
|
||||
embeddingsRecomputed,
|
||||
embeddingsFailed,
|
||||
};
|
||||
}
|
||||
|
||||
async search(
|
||||
|
|
@ -170,12 +169,14 @@ export class SlSearchService {
|
|||
minRrfScore = 0,
|
||||
): Promise<Array<{ sourceName: string; score: number; snippet?: string }>> {
|
||||
let queryEmbedding: number[] | null = null;
|
||||
try {
|
||||
queryEmbedding = await this.embeddingService.computeEmbedding(query);
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
`Failed to compute query embedding, falling back to FTS + trigram: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
if (this.embeddingService) {
|
||||
try {
|
||||
queryEmbedding = await this.embeddingService.computeEmbedding(query);
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
`Failed to compute query embedding, falling back to FTS + trigram: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const results = await this.slSourcesRepository.search(connectionId, queryEmbedding, query, limit, minRrfScore);
|
||||
|
|
|
|||
|
|
@ -105,6 +105,33 @@ describe('SqliteSlSourcesIndex', () => {
|
|||
expect(await index.search('finance', null, 'revenue', 10)).toEqual([]);
|
||||
});
|
||||
|
||||
it('clear removes sources and dictionary rows for one connection only', async () => {
|
||||
const index = new SqliteSlSourcesIndex({ dbPath });
|
||||
await index.upsertSources('warehouse', [
|
||||
{ sourceName: 'orders', searchText: 'orders revenue paid', embedding: null },
|
||||
]);
|
||||
await index.upsertSources('finance', [
|
||||
{ sourceName: 'invoices', searchText: 'invoices revenue paid', embedding: null },
|
||||
]);
|
||||
await index.replaceDictionaryEntries('warehouse', [
|
||||
{ connectionId: 'warehouse', sourceName: 'orders', columnName: 'status', value: 'paid', cardinality: 1 },
|
||||
]);
|
||||
await index.replaceDictionaryEntries('finance', [
|
||||
{ connectionId: 'finance', sourceName: 'invoices', columnName: 'status', value: 'paid', cardinality: 1 },
|
||||
]);
|
||||
|
||||
await expect(index.clear('warehouse')).resolves.toBe(1);
|
||||
|
||||
expect(await index.search('warehouse', null, 'revenue', 10)).toEqual([]);
|
||||
expect(await index.search('finance', null, 'revenue', 10)).toEqual([
|
||||
expect.objectContaining({ sourceName: 'invoices' }),
|
||||
]);
|
||||
await expect(index.searchDictionaryCandidates({ connectionIds: ['warehouse'], queryText: 'paid', limit: 10 }))
|
||||
.resolves.toEqual([]);
|
||||
await expect(index.searchDictionaryCandidates({ connectionIds: ['finance'], queryText: 'paid', limit: 10 }))
|
||||
.resolves.toEqual([expect.objectContaining({ connectionId: 'finance', sourceName: 'invoices' })]);
|
||||
});
|
||||
|
||||
it('returns lane candidates with stable connection-scoped IDs', async () => {
|
||||
const index = new SqliteSlSourcesIndex({ dbPath });
|
||||
|
||||
|
|
|
|||
|
|
@ -221,10 +221,9 @@ export class SqliteSlSourcesIndex implements SlSourcesIndexPort {
|
|||
);
|
||||
}
|
||||
|
||||
async deleteStale(connectionId: string, keepNames: string[]): Promise<void> {
|
||||
async deleteStale(connectionId: string, keepNames: string[]): Promise<number> {
|
||||
if (keepNames.length === 0) {
|
||||
await this.deleteByConnection(connectionId);
|
||||
return;
|
||||
return this.deleteByConnection(connectionId);
|
||||
}
|
||||
|
||||
const placeholders = keepNames.map(() => '?').join(', ');
|
||||
|
|
@ -257,18 +256,29 @@ export class SqliteSlSourcesIndex implements SlSourcesIndexPort {
|
|||
});
|
||||
|
||||
remove(stale.map((row) => row.source_name));
|
||||
return stale.length;
|
||||
}
|
||||
|
||||
async deleteByConnection(connectionId: string): Promise<void> {
|
||||
async deleteByConnection(connectionId: string): Promise<number> {
|
||||
return this.clear(connectionId);
|
||||
}
|
||||
|
||||
async clear(connectionId: string): Promise<number> {
|
||||
const rows = this.db
|
||||
.prepare('SELECT source_name FROM local_sl_sources WHERE connection_id = ?')
|
||||
.all(connectionId) as Array<{ source_name: string }>;
|
||||
const remove = this.db.transaction(() => {
|
||||
this.db.prepare('DELETE FROM local_sl_sources_fts WHERE connection_id = ?').run(connectionId);
|
||||
this.db.prepare('DELETE FROM local_sl_sources WHERE connection_id = ?').run(connectionId);
|
||||
this.db.prepare('DELETE FROM local_sl_dictionary_values_fts WHERE connection_id = ?').run(connectionId);
|
||||
this.db.prepare('DELETE FROM local_sl_dictionary_values WHERE connection_id = ?').run(connectionId);
|
||||
});
|
||||
remove();
|
||||
return rows.length;
|
||||
}
|
||||
|
||||
async deleteByConnectionAndName(connectionId: string, sourceName: string): Promise<void> {
|
||||
this.deleteByConnectionAndNameSync(connectionId, sourceName);
|
||||
async deleteByConnectionAndName(connectionId: string, sourceName: string): Promise<number> {
|
||||
return this.deleteByConnectionAndNameSync(connectionId, sourceName);
|
||||
}
|
||||
|
||||
async replaceDictionaryEntries(connectionId: string, entries: SlDictionaryEntry[]): Promise<void> {
|
||||
|
|
@ -537,7 +547,7 @@ export class SqliteSlSourcesIndex implements SlSourcesIndexPort {
|
|||
.filter((row) => row.rrfScore >= minRrfScore);
|
||||
}
|
||||
|
||||
private deleteByConnectionAndNameSync(connectionId: string, sourceName: string): void {
|
||||
private deleteByConnectionAndNameSync(connectionId: string, sourceName: string): number {
|
||||
const remove = this.db.transaction(() => {
|
||||
this.db
|
||||
.prepare(
|
||||
|
|
@ -548,7 +558,7 @@ export class SqliteSlSourcesIndex implements SlSourcesIndexPort {
|
|||
`,
|
||||
)
|
||||
.run(connectionId, sourceName);
|
||||
this.db
|
||||
const result = this.db
|
||||
.prepare(
|
||||
`
|
||||
DELETE FROM local_sl_sources
|
||||
|
|
@ -557,7 +567,8 @@ export class SqliteSlSourcesIndex implements SlSourcesIndexPort {
|
|||
`,
|
||||
)
|
||||
.run(connectionId, sourceName);
|
||||
return Number(result.changes);
|
||||
});
|
||||
remove();
|
||||
return remove();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,9 +4,9 @@ import { KnowledgeWikiService, type WikiFrontmatter } from './knowledge-wiki.ser
|
|||
function makeService() {
|
||||
const pagesRepository: Record<string, ReturnType<typeof vi.fn>> = {
|
||||
upsertPage: vi.fn().mockResolvedValue(undefined),
|
||||
deleteByKey: vi.fn().mockResolvedValue(undefined),
|
||||
deleteByScope: vi.fn().mockResolvedValue(undefined),
|
||||
deleteStale: vi.fn().mockResolvedValue(undefined),
|
||||
deleteByKey: vi.fn().mockResolvedValue(0),
|
||||
deleteByScope: vi.fn().mockResolvedValue(0),
|
||||
deleteStale: vi.fn().mockResolvedValue(0),
|
||||
getExistingSearchTexts: vi.fn().mockResolvedValue(new Map()),
|
||||
applyDiffTransactional: vi.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
|
|
@ -50,6 +50,87 @@ function makeService() {
|
|||
|
||||
const fm: WikiFrontmatter = { summary: 'sum', usage_mode: 'auto' };
|
||||
|
||||
describe('KnowledgeWikiService.syncIndex result stats', () => {
|
||||
it('reports scanned, updated, deleted, and embedding counts', async () => {
|
||||
const { service, pagesRepository, embeddingService, configService } = makeService();
|
||||
configService.listFiles.mockResolvedValue({ files: ['wiki/global/revenue.md'] });
|
||||
configService.readFile.mockResolvedValue({
|
||||
content: '---\nsummary: Revenue\nusage_mode: auto\ntags:\n - finance\n---\n\nPaid orders.\n',
|
||||
});
|
||||
pagesRepository.getExistingSearchTexts.mockResolvedValue(
|
||||
new Map([
|
||||
['old-page', { searchText: 'old', hasEmbedding: true }],
|
||||
]),
|
||||
);
|
||||
embeddingService.computeEmbeddingsBulk.mockResolvedValue([[0.1, 0.2, 0.3]]);
|
||||
pagesRepository.deleteStale.mockResolvedValue(1);
|
||||
|
||||
await expect(service.syncIndex('GLOBAL', null)).resolves.toEqual({
|
||||
scanned: 1,
|
||||
updated: 1,
|
||||
deleted: 1,
|
||||
embeddingsRecomputed: 1,
|
||||
embeddingsFailed: 0,
|
||||
});
|
||||
});
|
||||
|
||||
it('indexes lexical rows when embeddings are not configured', async () => {
|
||||
const { pagesRepository, configService, gitService, logger } = makeService();
|
||||
const service = new KnowledgeWikiService(
|
||||
configService as any,
|
||||
null,
|
||||
pagesRepository as any,
|
||||
gitService as any,
|
||||
logger as any,
|
||||
);
|
||||
configService.listFiles.mockResolvedValue({ files: ['wiki/global/revenue.md'] });
|
||||
configService.readFile.mockResolvedValue({
|
||||
content: '---\nsummary: Revenue\nusage_mode: auto\n---\n\nPaid orders.\n',
|
||||
});
|
||||
pagesRepository.getExistingSearchTexts.mockResolvedValue(new Map());
|
||||
pagesRepository.deleteStale.mockResolvedValue(0);
|
||||
|
||||
const result = await service.syncIndex('GLOBAL', null);
|
||||
|
||||
expect(result.embeddingsRecomputed).toBe(0);
|
||||
expect(result.embeddingsFailed).toBe(0);
|
||||
expect(pagesRepository.upsertPage).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ pageKey: 'revenue', embedding: null }),
|
||||
);
|
||||
});
|
||||
|
||||
it('does not update unchanged lexical-only wiki rows on repeated sync', async () => {
|
||||
const { pagesRepository, configService, gitService, logger } = makeService();
|
||||
const service = new KnowledgeWikiService(
|
||||
configService as any,
|
||||
null,
|
||||
pagesRepository as any,
|
||||
gitService as any,
|
||||
logger as any,
|
||||
);
|
||||
configService.listFiles.mockResolvedValue({ files: ['wiki/global/revenue.md'] });
|
||||
configService.readFile.mockResolvedValue({
|
||||
content: '---\nsummary: Revenue\nusage_mode: auto\n---\n\nPaid orders.\n',
|
||||
});
|
||||
pagesRepository.getExistingSearchTexts.mockResolvedValue(
|
||||
new Map([
|
||||
['revenue', { searchText: 'revenue\nRevenue\nPaid orders.', hasEmbedding: false }],
|
||||
]),
|
||||
);
|
||||
pagesRepository.deleteStale.mockResolvedValue(0);
|
||||
|
||||
await expect(service.syncIndex('GLOBAL', null)).resolves.toEqual({
|
||||
scanned: 1,
|
||||
updated: 0,
|
||||
deleted: 0,
|
||||
embeddingsRecomputed: 0,
|
||||
embeddingsFailed: 0,
|
||||
});
|
||||
expect(pagesRepository.upsertPage).not.toHaveBeenCalled();
|
||||
expect(pagesRepository.deleteStale).toHaveBeenCalledWith('GLOBAL', null, ['revenue']);
|
||||
});
|
||||
});
|
||||
|
||||
describe('KnowledgeWikiService.forWorktree isolation', () => {
|
||||
it('syncSinglePage in worktree scope does not call pagesRepository.upsertPage', async () => {
|
||||
const { service, pagesRepository, embeddingService } = makeService();
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import { createHash } from 'node:crypto';
|
|||
import YAML from 'yaml';
|
||||
import type { KtxEmbeddingPort, KtxFileStorePort, KtxLogger } from '../core/index.js';
|
||||
import { noopLogger } from '../core/index.js';
|
||||
import type { ReindexWorkResult } from '../index-sync/types.js';
|
||||
import { assertFlatWikiKey, isFlatWikiKey } from './keys.js';
|
||||
import { buildKnowledgeSearchText } from './knowledge-search-text.js';
|
||||
import type { KnowledgeGitDiffPort, KnowledgeIndexPort, UpsertPageParams } from './ports.js';
|
||||
|
|
@ -16,7 +17,7 @@ export class KnowledgeWikiService {
|
|||
|
||||
constructor(
|
||||
private readonly configService: KtxFileStorePort,
|
||||
private readonly embeddingService: KtxEmbeddingPort,
|
||||
private readonly embeddingService: KtxEmbeddingPort | null,
|
||||
private readonly pagesRepository: KnowledgeIndexPort,
|
||||
private readonly gitService: KnowledgeGitDiffPort,
|
||||
private readonly logger: KtxLogger = noopLogger,
|
||||
|
|
@ -246,10 +247,12 @@ export class KnowledgeWikiService {
|
|||
const searchText = buildKnowledgeSearchText(pageKey, frontmatter.summary, content, frontmatter.tags);
|
||||
|
||||
let embedding: number[] | null = null;
|
||||
try {
|
||||
embedding = await this.embeddingService.computeEmbedding(searchText);
|
||||
} catch (err) {
|
||||
this.logger.warn(`Embedding failed for page "${pageKey}": ${err instanceof Error ? err.message : String(err)}`);
|
||||
if (this.embeddingService) {
|
||||
try {
|
||||
embedding = await this.embeddingService.computeEmbedding(searchText);
|
||||
} catch (err) {
|
||||
this.logger.warn(`Embedding failed for page "${pageKey}": ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
}
|
||||
|
||||
await this.pagesRepository.upsertPage({
|
||||
|
|
@ -269,14 +272,21 @@ export class KnowledgeWikiService {
|
|||
* Full sync: load all pages from disk for a scope, reindex changed pages, clean stale entries.
|
||||
* Mirrors SlSearchService.indexSources() pattern.
|
||||
*/
|
||||
async syncIndex(scope: string, scopeId?: string | null): Promise<void> {
|
||||
async syncIndex(scope: string, scopeId?: string | null): Promise<ReindexWorkResult> {
|
||||
const pageKeys = await this.listPageKeys(scope, scopeId);
|
||||
const existing = await this.pagesRepository.getExistingSearchTexts(scope, scopeId ?? null);
|
||||
|
||||
if (pageKeys.length === 0) {
|
||||
await this.pagesRepository.deleteByScope(scope, scopeId ?? null);
|
||||
return;
|
||||
const deleted = await this.pagesRepository.deleteByScope(scope, scopeId ?? null);
|
||||
return {
|
||||
scanned: 0,
|
||||
updated: 0,
|
||||
deleted,
|
||||
embeddingsRecomputed: 0,
|
||||
embeddingsFailed: 0,
|
||||
};
|
||||
}
|
||||
|
||||
// Load and parse all pages
|
||||
const pages: Array<{ pageKey: string; frontmatter: WikiFrontmatter; content: string; searchText: string }> = [];
|
||||
for (const key of pageKeys) {
|
||||
const page = await this.readPage(scope, scopeId, key);
|
||||
|
|
@ -286,58 +296,58 @@ export class KnowledgeWikiService {
|
|||
}
|
||||
}
|
||||
|
||||
// Detect changes
|
||||
const existing = await this.pagesRepository.getExistingSearchTexts(scope, scopeId ?? null);
|
||||
const changedPages = pages.filter((p) => {
|
||||
const ex = existing.get(p.pageKey);
|
||||
return !ex || ex.searchText !== p.searchText || !ex.hasEmbedding;
|
||||
const embeddingService = this.embeddingService;
|
||||
const changedPages = pages.filter((page) => {
|
||||
const previous = existing.get(page.pageKey);
|
||||
return (
|
||||
!previous ||
|
||||
previous.searchText !== page.searchText ||
|
||||
(embeddingService !== null && !previous.hasEmbedding)
|
||||
);
|
||||
});
|
||||
|
||||
if (changedPages.length === 0) {
|
||||
// Still clean up stale
|
||||
await this.pagesRepository.deleteStale(scope, scopeId ?? null, pageKeys);
|
||||
this.logger.log(`Wiki sync ${scope}: all ${pages.length} pages up to date`);
|
||||
return;
|
||||
}
|
||||
let embeddings: (number[] | null)[] = changedPages.map(() => null);
|
||||
let embeddingsRecomputed = 0;
|
||||
let embeddingsFailed = 0;
|
||||
|
||||
// Compute embeddings for changed pages (batched)
|
||||
const changedTexts = changedPages.map((p) => p.searchText);
|
||||
let embeddings: (number[] | null)[];
|
||||
try {
|
||||
const batchSize = this.embeddingService.maxBatchSize;
|
||||
const all: number[][] = [];
|
||||
for (let i = 0; i < changedTexts.length; i += batchSize) {
|
||||
const batch = changedTexts.slice(i, i + batchSize);
|
||||
const batchEmb = await this.embeddingService.computeEmbeddingsBulk(batch);
|
||||
all.push(...batchEmb);
|
||||
if (embeddingService && changedPages.length > 0) {
|
||||
try {
|
||||
const changedTexts = changedPages.map((page) => page.searchText);
|
||||
const all: number[][] = [];
|
||||
for (let i = 0; i < changedTexts.length; i += embeddingService.maxBatchSize) {
|
||||
const batch = changedTexts.slice(i, i + embeddingService.maxBatchSize);
|
||||
all.push(...(await embeddingService.computeEmbeddingsBulk(batch)));
|
||||
}
|
||||
embeddings = all;
|
||||
embeddingsRecomputed = all.length;
|
||||
} catch (err) {
|
||||
this.logger.warn(`Embedding batch failed during sync: ${err instanceof Error ? err.message : String(err)}`);
|
||||
embeddingsFailed = changedPages.length;
|
||||
}
|
||||
embeddings = all;
|
||||
} catch (err) {
|
||||
this.logger.warn(`Embedding batch failed during sync: ${err instanceof Error ? err.message : String(err)}`);
|
||||
embeddings = changedPages.map(() => null);
|
||||
}
|
||||
|
||||
// Upsert changed pages
|
||||
for (let i = 0; i < changedPages.length; i++) {
|
||||
const p = changedPages[i];
|
||||
for (let i = 0; i < changedPages.length; i += 1) {
|
||||
const page = changedPages[i]!;
|
||||
await this.pagesRepository.upsertPage({
|
||||
scope,
|
||||
scopeId: scopeId ?? null,
|
||||
pageKey: p.pageKey,
|
||||
summary: p.frontmatter.summary,
|
||||
usageMode: p.frontmatter.usage_mode,
|
||||
sortOrder: p.frontmatter.sort_order ?? 0,
|
||||
searchText: p.searchText,
|
||||
embedding: embeddings[i],
|
||||
pageKey: page.pageKey,
|
||||
summary: page.frontmatter.summary,
|
||||
usageMode: page.frontmatter.usage_mode,
|
||||
sortOrder: page.frontmatter.sort_order ?? 0,
|
||||
searchText: page.searchText,
|
||||
embedding: embeddings[i] ?? null,
|
||||
});
|
||||
}
|
||||
|
||||
// Clean stale entries
|
||||
await this.pagesRepository.deleteStale(scope, scopeId ?? null, pageKeys);
|
||||
|
||||
this.logger.log(
|
||||
`Wiki sync ${scope}: ${changedPages.length}/${pages.length} reindexed, ${pages.length - changedPages.length} unchanged`,
|
||||
);
|
||||
const deleted = await this.pagesRepository.deleteStale(scope, scopeId ?? null, pageKeys);
|
||||
return {
|
||||
scanned: pages.length,
|
||||
updated: changedPages.length,
|
||||
deleted,
|
||||
embeddingsRecomputed,
|
||||
embeddingsFailed,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -388,12 +398,14 @@ export class KnowledgeWikiService {
|
|||
parsed.frontmatter.tags,
|
||||
);
|
||||
let embedding: number[] | null = null;
|
||||
try {
|
||||
embedding = await this.embeddingService.computeEmbedding(searchText);
|
||||
} catch (err) {
|
||||
this.logger.warn(
|
||||
`[wiki.sync] embedding failed for ${parsedPath.pageKey}: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
if (this.embeddingService) {
|
||||
try {
|
||||
embedding = await this.embeddingService.computeEmbedding(searchText);
|
||||
} catch (err) {
|
||||
this.logger.warn(
|
||||
`[wiki.sync] embedding failed for ${parsedPath.pageKey}: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
const contentHash = createHash('sha256').update(content).digest('hex');
|
||||
upserts.push({
|
||||
|
|
|
|||
|
|
@ -33,9 +33,9 @@ export interface KnowledgeIndexPort {
|
|||
scope: string,
|
||||
scopeId: string | null,
|
||||
): Promise<Map<string, { searchText: string; hasEmbedding: boolean }>>;
|
||||
deleteStale(scope: string, scopeId: string | null, keepKeys: string[]): Promise<void>;
|
||||
deleteByScope(scope: string, scopeId: string | null): Promise<void>;
|
||||
deleteByKey(scope: string, scopeId: string | null, pageKey: string): Promise<void>;
|
||||
deleteStale(scope: string, scopeId: string | null, keepKeys: string[]): Promise<number>;
|
||||
deleteByScope(scope: string, scopeId: string | null): Promise<number>;
|
||||
deleteByKey(scope: string, scopeId: string | null, pageKey: string): Promise<number>;
|
||||
findPageByKey(
|
||||
scope: string,
|
||||
scopeId: string | null,
|
||||
|
|
|
|||
|
|
@ -65,6 +65,35 @@ describe('SqliteKnowledgeIndex', () => {
|
|||
expect(index.search('churn', 10)).toEqual([]);
|
||||
});
|
||||
|
||||
it('clear removes one wiki scope and leaves other scopes intact', async () => {
|
||||
const index = new SqliteKnowledgeIndex({ dbPath });
|
||||
index.sync([
|
||||
page({ path: 'wiki/global/revenue.md', key: 'revenue', scope: 'GLOBAL', scopeId: null }),
|
||||
page({
|
||||
path: 'wiki/user/local/revenue.md',
|
||||
key: 'revenue',
|
||||
scope: 'USER',
|
||||
scopeId: 'local',
|
||||
summary: 'Local revenue',
|
||||
content: 'Local revenue notes.',
|
||||
}),
|
||||
page({
|
||||
path: 'wiki/user/alex/revenue.md',
|
||||
key: 'revenue',
|
||||
scope: 'USER',
|
||||
scopeId: 'alex',
|
||||
summary: 'Alex revenue',
|
||||
content: 'Alex revenue notes.',
|
||||
}),
|
||||
]);
|
||||
|
||||
expect(index.clear('USER', 'local')).toBe(1);
|
||||
|
||||
expect(index.search('Local', 10)).toEqual([]);
|
||||
expect(index.search('Alex', 10)).toEqual([expect.objectContaining({ path: 'wiki/user/alex/revenue.md' })]);
|
||||
expect(index.search('definition', 10)).toEqual([expect.objectContaining({ path: 'wiki/global/revenue.md' })]);
|
||||
});
|
||||
|
||||
it('exposes existing search text and embedding state for incremental refresh', () => {
|
||||
const index = new SqliteKnowledgeIndex({ dbPath });
|
||||
index.sync([page({ path: 'wiki/global/revenue.md', key: 'revenue', embedding: [1, 0] })]);
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import { dirname } from 'node:path';
|
|||
import Database from 'better-sqlite3';
|
||||
import { buildKnowledgeSearchText } from './knowledge-search-text.js';
|
||||
import type { LocalKnowledgeScope } from './local-knowledge.js';
|
||||
import type { KnowledgeIndexPageListing, UpsertPageParams } from './ports.js';
|
||||
|
||||
export interface SqliteKnowledgeIndexOptions {
|
||||
dbPath: string;
|
||||
|
|
@ -12,6 +13,7 @@ export interface SqliteKnowledgeIndexPage {
|
|||
path: string;
|
||||
key: string;
|
||||
scope: LocalKnowledgeScope;
|
||||
scopeId?: string | null;
|
||||
summary: string;
|
||||
content: string;
|
||||
tags: string[];
|
||||
|
|
@ -106,6 +108,7 @@ export class SqliteKnowledgeIndex {
|
|||
path TEXT PRIMARY KEY,
|
||||
key TEXT NOT NULL,
|
||||
scope TEXT NOT NULL,
|
||||
scope_id TEXT,
|
||||
summary TEXT NOT NULL,
|
||||
content TEXT NOT NULL,
|
||||
tags TEXT NOT NULL,
|
||||
|
|
@ -129,6 +132,9 @@ export class SqliteKnowledgeIndex {
|
|||
if (!columnNames.has('embedding_json')) {
|
||||
this.db.exec('ALTER TABLE knowledge_pages ADD COLUMN embedding_json TEXT');
|
||||
}
|
||||
if (!columnNames.has('scope_id')) {
|
||||
this.db.exec('ALTER TABLE knowledge_pages ADD COLUMN scope_id TEXT');
|
||||
}
|
||||
}
|
||||
|
||||
sync(pages: SqliteKnowledgeIndexPage[]): void {
|
||||
|
|
@ -142,11 +148,12 @@ export class SqliteKnowledgeIndex {
|
|||
? this.db.prepare('DELETE FROM knowledge_pages_fts')
|
||||
: this.db.prepare(`DELETE FROM knowledge_pages_fts WHERE path NOT IN (${keepPaths.map(() => '?').join(', ')})`);
|
||||
const upsertPage = this.db.prepare(`
|
||||
INSERT INTO knowledge_pages (path, key, scope, summary, content, tags, search_text, embedding_json)
|
||||
VALUES (@path, @key, @scope, @summary, @content, @tags, @searchText, @embeddingJson)
|
||||
INSERT INTO knowledge_pages (path, key, scope, scope_id, summary, content, tags, search_text, embedding_json)
|
||||
VALUES (@path, @key, @scope, @scopeId, @summary, @content, @tags, @searchText, @embeddingJson)
|
||||
ON CONFLICT(path) DO UPDATE SET
|
||||
key = excluded.key,
|
||||
scope = excluded.scope,
|
||||
scope_id = excluded.scope_id,
|
||||
summary = excluded.summary,
|
||||
content = excluded.content,
|
||||
tags = excluded.tags,
|
||||
|
|
@ -168,6 +175,7 @@ export class SqliteKnowledgeIndex {
|
|||
path: page.path,
|
||||
key: page.key,
|
||||
scope: page.scope,
|
||||
scopeId: page.scopeId ?? null,
|
||||
summary: page.summary,
|
||||
content: searchText,
|
||||
tags: page.tags.join(' '),
|
||||
|
|
@ -275,4 +283,201 @@ export class SqliteKnowledgeIndex {
|
|||
score: scoreFromRank(row.rawScore),
|
||||
}));
|
||||
}
|
||||
|
||||
private pathForPage(scope: string, scopeId: string | null, pageKey: string): string {
|
||||
return scope === 'GLOBAL' ? `wiki/global/${pageKey}.md` : `wiki/user/${scopeId ?? 'local'}/${pageKey}.md`;
|
||||
}
|
||||
|
||||
async upsertPage(params: UpsertPageParams): Promise<void> {
|
||||
const path = this.pathForPage(params.scope, params.scopeId, params.pageKey);
|
||||
const row = {
|
||||
path,
|
||||
key: params.pageKey,
|
||||
scope: params.scope,
|
||||
scopeId: params.scopeId,
|
||||
summary: params.summary,
|
||||
content: params.searchText,
|
||||
tags: '',
|
||||
searchText: params.searchText,
|
||||
embeddingJson: params.embedding && params.embedding.length > 0 ? JSON.stringify(params.embedding) : null,
|
||||
};
|
||||
const write = this.db.transaction(() => {
|
||||
this.db
|
||||
.prepare(
|
||||
`
|
||||
INSERT INTO knowledge_pages (path, key, scope, scope_id, summary, content, tags, search_text, embedding_json)
|
||||
VALUES (@path, @key, @scope, @scopeId, @summary, @content, @tags, @searchText, @embeddingJson)
|
||||
ON CONFLICT(path) DO UPDATE SET
|
||||
key = excluded.key,
|
||||
scope = excluded.scope,
|
||||
scope_id = excluded.scope_id,
|
||||
summary = excluded.summary,
|
||||
content = excluded.content,
|
||||
tags = excluded.tags,
|
||||
search_text = excluded.search_text,
|
||||
embedding_json = excluded.embedding_json
|
||||
`,
|
||||
)
|
||||
.run(row);
|
||||
this.db.prepare('DELETE FROM knowledge_pages_fts WHERE path = @path').run(row);
|
||||
this.db
|
||||
.prepare(
|
||||
`
|
||||
INSERT INTO knowledge_pages_fts (path, key, summary, content, tags)
|
||||
VALUES (@path, @key, @summary, @content, @tags)
|
||||
`,
|
||||
)
|
||||
.run(row);
|
||||
});
|
||||
write();
|
||||
}
|
||||
|
||||
async getExistingSearchTexts(
|
||||
scope: string,
|
||||
scopeId: string | null,
|
||||
): Promise<Map<string, { searchText: string; hasEmbedding: boolean }>> {
|
||||
const rows = this.db
|
||||
.prepare(
|
||||
`
|
||||
SELECT key, search_text, embedding_json
|
||||
FROM knowledge_pages
|
||||
WHERE scope = ?
|
||||
AND scope_id IS ?
|
||||
ORDER BY key ASC
|
||||
`,
|
||||
)
|
||||
.all(scope, scopeId) as Array<{ key: string; search_text: string; embedding_json: string | null }>;
|
||||
return new Map(
|
||||
rows.map((row) => [row.key, { searchText: row.search_text, hasEmbedding: row.embedding_json !== null }]),
|
||||
);
|
||||
}
|
||||
|
||||
async deleteStale(scope: string, scopeId: string | null, keepKeys: string[]): Promise<number> {
|
||||
if (keepKeys.length === 0) {
|
||||
return this.deleteByScope(scope, scopeId);
|
||||
}
|
||||
const placeholders = keepKeys.map(() => '?').join(', ');
|
||||
const stale = this.db
|
||||
.prepare(
|
||||
`
|
||||
SELECT key
|
||||
FROM knowledge_pages
|
||||
WHERE scope = ?
|
||||
AND scope_id IS ?
|
||||
AND key NOT IN (${placeholders})
|
||||
`,
|
||||
)
|
||||
.all(scope, scopeId, ...keepKeys) as Array<{ key: string }>;
|
||||
for (const row of stale) {
|
||||
await this.deleteByKey(scope, scopeId, row.key);
|
||||
}
|
||||
return stale.length;
|
||||
}
|
||||
|
||||
async deleteByScope(scope: string, scopeId: string | null): Promise<number> {
|
||||
return this.clear(scope, scopeId);
|
||||
}
|
||||
|
||||
async deleteByKey(scope: string, scopeId: string | null, pageKey: string): Promise<number> {
|
||||
const path = this.pathForPage(scope, scopeId, pageKey);
|
||||
const remove = this.db.transaction(() => {
|
||||
this.db.prepare('DELETE FROM knowledge_pages_fts WHERE path = ?').run(path);
|
||||
const result = this.db.prepare('DELETE FROM knowledge_pages WHERE path = ?').run(path);
|
||||
return Number(result.changes);
|
||||
});
|
||||
return remove();
|
||||
}
|
||||
|
||||
clear(scope: string, scopeId: string | null): number {
|
||||
const rows = this.db
|
||||
.prepare('SELECT path FROM knowledge_pages WHERE scope = ? AND scope_id IS ?')
|
||||
.all(scope, scopeId) as Array<{ path: string }>;
|
||||
const remove = this.db.transaction((paths: string[]) => {
|
||||
for (const path of paths) {
|
||||
this.db.prepare('DELETE FROM knowledge_pages_fts WHERE path = ?').run(path);
|
||||
this.db.prepare('DELETE FROM knowledge_pages WHERE path = ?').run(path);
|
||||
}
|
||||
});
|
||||
remove(rows.map((row) => row.path));
|
||||
return rows.length;
|
||||
}
|
||||
|
||||
async applyDiffTransactional(params: {
|
||||
runId: string;
|
||||
upserts: UpsertPageParams[];
|
||||
deletes: Array<{ scope: string; scopeId: string | null; pageKey: string }>;
|
||||
}): Promise<void> {
|
||||
void params.runId;
|
||||
for (const page of params.upserts) {
|
||||
await this.upsertPage(page);
|
||||
}
|
||||
for (const page of params.deletes) {
|
||||
await this.deleteByKey(page.scope, page.scopeId, page.pageKey);
|
||||
}
|
||||
}
|
||||
|
||||
async findPageByKey(
|
||||
scope: string,
|
||||
scopeId: string | null,
|
||||
pageKey: string,
|
||||
): Promise<{ id?: string; page_key: string } | null> {
|
||||
const path = this.pathForPage(scope, scopeId, pageKey);
|
||||
const row = this.db.prepare('SELECT path, key FROM knowledge_pages WHERE path = ?').get(path) as
|
||||
| { path: string; key: string }
|
||||
| undefined;
|
||||
return row ? { id: row.path, page_key: row.key } : null;
|
||||
}
|
||||
|
||||
async listPagesForUser(userId: string): Promise<KnowledgeIndexPageListing[]> {
|
||||
const rows = this.db
|
||||
.prepare(
|
||||
`
|
||||
SELECT path, key, scope, scope_id, summary, tags
|
||||
FROM knowledge_pages
|
||||
WHERE scope = 'GLOBAL'
|
||||
OR (scope = 'USER' AND scope_id = ?)
|
||||
ORDER BY scope ASC, key ASC
|
||||
`,
|
||||
)
|
||||
.all(userId) as Array<{
|
||||
path: string;
|
||||
key: string;
|
||||
scope: string;
|
||||
scope_id: string | null;
|
||||
summary: string;
|
||||
tags: string;
|
||||
}>;
|
||||
return rows.map((row) => ({
|
||||
id: row.path,
|
||||
page_key: row.key,
|
||||
summary: row.summary,
|
||||
scope: row.scope,
|
||||
scope_id: row.scope_id,
|
||||
tags: row.tags.split(/\s+/).filter(Boolean),
|
||||
}));
|
||||
}
|
||||
|
||||
async getUserPageCount(userId: string): Promise<number> {
|
||||
const row = this.db
|
||||
.prepare("SELECT COUNT(*) AS count FROM knowledge_pages WHERE scope = 'USER' AND scope_id = ?")
|
||||
.get(userId) as { count: number };
|
||||
return row.count;
|
||||
}
|
||||
|
||||
async incrementUsageCount(): Promise<void> {}
|
||||
|
||||
async searchRRF(
|
||||
userId: string,
|
||||
_embedding: number[] | null,
|
||||
queryText: string,
|
||||
limit: number,
|
||||
): Promise<Array<{ pageKey: string; summary: string; rrfScore: number }>> {
|
||||
const allowedPages = new Map((await this.listPagesForUser(userId)).map((page) => [page.id, page]));
|
||||
return this.search(queryText, limit)
|
||||
.map((row) => {
|
||||
const page = allowedPages.get(row.path);
|
||||
return page ? { pageKey: page.page_key, summary: page.summary, rrfScore: row.score } : null;
|
||||
})
|
||||
.filter((row): row is { pageKey: string; summary: string; rrfScore: number } => row !== null);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ const identifierAllowPatterns = [
|
|||
/^packages\/cli\/src\/(?:index|managed-local-embeddings|managed-python-command|managed-python-daemon|managed-python-runtime|release-version|runtime)(?:\.test)?\.ts$/,
|
||||
/^python\/ktx-daemon\/src\/ktx_daemon\/__init__\.py$/,
|
||||
/^scripts\/(?:build-public-npm-package|build-python-runtime-wheel|local-embeddings-runtime-smoke|package-artifacts|public-npm-release-metadata|published-package-smoke|release-readiness)(?:\.test)?\.mjs$/,
|
||||
/^scripts\/semantic-release-config\.cjs$/,
|
||||
];
|
||||
const forbiddenIdentifierTerms = ['kae' + 'lio', 'Kae' + 'lio', 'KAE' + 'LIO_'];
|
||||
|
||||
|
|
|
|||
|
|
@ -79,6 +79,7 @@ describe('scanFileContent', () => {
|
|||
assert.equal(scanFileContent('scripts/local-embeddings-runtime-smoke.mjs', `@${name}/ktx`).length, 0);
|
||||
assert.equal(scanFileContent('scripts/package-artifacts.test.mjs', `${name}-ktx`).length, 0);
|
||||
assert.equal(scanFileContent('scripts/public-npm-release-metadata.mjs', `@${name}/ktx`).length, 0);
|
||||
assert.equal(scanFileContent('scripts/semantic-release-config.cjs', `${name}-ktx-`).length, 0);
|
||||
assert.equal(scanFileContent('packages/cli/src/release-version.ts', `@${name}/ktx`).length, 0);
|
||||
assert.equal(scanFileContent('packages/cli/src/managed-python-runtime.ts', `${name}_ktx`).length, 0);
|
||||
assert.equal(scanFileContent('python/ktx-daemon/src/ktx_daemon/__init__.py', `${name}-ktx`).length, 0);
|
||||
|
|
|
|||
|
|
@ -193,12 +193,12 @@ describe('standalone example docs', () => {
|
|||
|
||||
assert.match(rootReadme, publicPackagePattern('npm install -g {package}'));
|
||||
assert.match(quickstart, publicPackagePattern('npm install -g {package}'));
|
||||
assert.match(quickstart, /ktx dev runtime install --feature local-embeddings --yes/);
|
||||
assert.match(quickstart, /ktx dev runtime start --feature local-embeddings/);
|
||||
assert.match(quickstart, /ktx admin runtime install --feature local-embeddings --yes/);
|
||||
assert.match(quickstart, /ktx admin runtime start --feature local-embeddings/);
|
||||
assert.match(packageArtifacts, /requires `uv` on `PATH`/);
|
||||
assert.match(packageArtifacts, /ktx dev runtime status/);
|
||||
assert.match(packageArtifacts, /ktx dev runtime status/);
|
||||
assert.doesNotMatch(packageArtifacts, /ktx dev runtime prune/);
|
||||
assert.match(packageArtifacts, /ktx admin runtime status/);
|
||||
assert.match(packageArtifacts, /ktx admin runtime status/);
|
||||
assert.doesNotMatch(packageArtifacts, /ktx admin runtime prune/);
|
||||
assert.match(
|
||||
packageArtifacts,
|
||||
new RegExp(
|
||||
|
|
@ -229,9 +229,9 @@ describe('standalone example docs', () => {
|
|||
assert.doesNotMatch(readme, /standalone Python distributions/);
|
||||
assert.doesNotMatch(readme, /installs the Python artifacts directly/);
|
||||
assert.match(readme, /requires `uv` on `PATH`/);
|
||||
assert.match(readme, /ktx dev runtime status/);
|
||||
assert.match(readme, /ktx dev runtime status/);
|
||||
assert.doesNotMatch(readme, /ktx dev runtime prune/);
|
||||
assert.match(readme, /ktx admin runtime status/);
|
||||
assert.match(readme, /ktx admin runtime status/);
|
||||
assert.doesNotMatch(readme, /ktx admin runtime prune/);
|
||||
assert.doesNotMatch(readme, /@ktx\/context/);
|
||||
assert.doesNotMatch(readme, /@ktx\/cli/);
|
||||
assert.doesNotMatch(readme, /python -m ktx_daemon semantic-validate/);
|
||||
|
|
@ -241,7 +241,7 @@ describe('standalone example docs', () => {
|
|||
const rootReadme = await readText('README.md');
|
||||
const cliMeta = await readText('docs-site/content/docs/cli-reference/meta.json');
|
||||
const ingestReference = await readText('docs-site/content/docs/cli-reference/ktx-ingest.mdx');
|
||||
const devReference = await readText('docs-site/content/docs/cli-reference/ktx-dev.mdx');
|
||||
const adminReference = await readText('docs-site/content/docs/cli-reference/ktx-admin.mdx');
|
||||
const setupReference = await readText('docs-site/content/docs/cli-reference/ktx-setup.mdx');
|
||||
const buildingContext = await readText('docs-site/content/docs/guides/building-context.mdx');
|
||||
const contextSources = await readText('docs-site/content/docs/integrations/context-sources.mdx');
|
||||
|
|
@ -275,7 +275,7 @@ describe('standalone example docs', () => {
|
|||
assert.doesNotMatch(ingestReference, /--adapter/);
|
||||
assert.doesNotMatch(ingestReference, /ktx ingest watch/);
|
||||
assert.doesNotMatch(ingestReference, /live-database/);
|
||||
assert.doesNotMatch(devReference, /ktx scan/);
|
||||
assert.doesNotMatch(adminReference, /ktx scan/);
|
||||
assert.doesNotMatch(buildingContext, /ktx ingest watch/);
|
||||
assert.doesNotMatch(buildingContext, /ktx ingest status/);
|
||||
assert.doesNotMatch(buildingContext, /ktx ingest replay/);
|
||||
|
|
|
|||
|
|
@ -237,17 +237,17 @@ function parseDaemonBaseUrl(stdout) {
|
|||
}
|
||||
|
||||
async function startDaemon(cleanInstallDir) {
|
||||
const result = await run('pnpm', ['exec', 'ktx', 'dev', 'runtime', 'start'], {
|
||||
const result = await run('pnpm', ['exec', 'ktx', 'admin', 'runtime', 'start'], {
|
||||
cwd: cleanInstallDir,
|
||||
env: managedRuntimeEnv(cleanInstallDir),
|
||||
timeout: 120_000,
|
||||
});
|
||||
requireSuccess('ktx dev runtime start', result);
|
||||
requireSuccess('ktx admin runtime start', result);
|
||||
return parseDaemonBaseUrl(result.stdout);
|
||||
}
|
||||
|
||||
async function stopDaemon(cleanInstallDir) {
|
||||
await run('pnpm', ['exec', 'ktx', 'dev', 'runtime', 'stop'], {
|
||||
await run('pnpm', ['exec', 'ktx', 'admin', 'runtime', 'stop'], {
|
||||
cwd: cleanInstallDir,
|
||||
env: managedRuntimeEnv(cleanInstallDir),
|
||||
timeout: 30_000,
|
||||
|
|
@ -271,7 +271,7 @@ async function prepareCleanInstall(layout, cleanInstallDir) {
|
|||
await run('pnpm', ['install'], { cwd: cleanInstallDir, timeout: 120_000 }).then((result) =>
|
||||
requireSuccess('pnpm install clean artifact project', result),
|
||||
);
|
||||
await run('pnpm', ['exec', 'ktx', 'dev', 'runtime', 'install', '--yes'], {
|
||||
await run('pnpm', ['exec', 'ktx', 'admin', 'runtime', 'install', '--yes'], {
|
||||
cwd: cleanInstallDir,
|
||||
env: managedRuntimeEnv(cleanInstallDir),
|
||||
timeout: 120_000,
|
||||
|
|
|
|||
|
|
@ -74,27 +74,27 @@ export function localEmbeddingsSmokeCommands(input) {
|
|||
timeoutMs: 60_000,
|
||||
},
|
||||
{
|
||||
label: 'ktx dev runtime status missing',
|
||||
label: 'ktx admin runtime status missing',
|
||||
command: 'pnpm',
|
||||
args: ['exec', 'ktx', 'dev', 'runtime', 'status', '--json'],
|
||||
args: ['exec', 'ktx', 'admin', 'runtime', 'status', '--json'],
|
||||
timeoutMs: 60_000,
|
||||
},
|
||||
{
|
||||
label: 'ktx dev runtime install local embeddings',
|
||||
label: 'ktx admin runtime install local embeddings',
|
||||
command: 'pnpm',
|
||||
args: ['exec', 'ktx', 'dev', 'runtime', 'install', '--feature', 'local-embeddings', '--yes'],
|
||||
args: ['exec', 'ktx', 'admin', 'runtime', 'install', '--feature', 'local-embeddings', '--yes'],
|
||||
timeoutMs: 1_200_000,
|
||||
},
|
||||
{
|
||||
label: 'ktx dev runtime status local embeddings ready',
|
||||
label: 'ktx admin runtime status local embeddings ready',
|
||||
command: 'pnpm',
|
||||
args: ['exec', 'ktx', 'dev', 'runtime', 'status', '--json'],
|
||||
args: ['exec', 'ktx', 'admin', 'runtime', 'status', '--json'],
|
||||
timeoutMs: 60_000,
|
||||
},
|
||||
{
|
||||
label: 'ktx dev runtime start local embeddings',
|
||||
label: 'ktx admin runtime start local embeddings',
|
||||
command: 'pnpm',
|
||||
args: ['exec', 'ktx', 'dev', 'runtime', 'start', '--feature', 'local-embeddings'],
|
||||
args: ['exec', 'ktx', 'admin', 'runtime', 'start', '--feature', 'local-embeddings'],
|
||||
timeoutMs: 300_000,
|
||||
},
|
||||
{
|
||||
|
|
@ -118,9 +118,9 @@ export function localEmbeddingsSmokeCommands(input) {
|
|||
timeoutMs: 900_000,
|
||||
},
|
||||
{
|
||||
label: 'ktx dev runtime stop local embeddings',
|
||||
label: 'ktx admin runtime stop local embeddings',
|
||||
command: 'pnpm',
|
||||
args: ['exec', 'ktx', 'dev', 'runtime', 'stop'],
|
||||
args: ['exec', 'ktx', 'admin', 'runtime', 'stop'],
|
||||
timeoutMs: 60_000,
|
||||
},
|
||||
];
|
||||
|
|
@ -374,7 +374,7 @@ export async function runLocalEmbeddingsRuntimeSmoke(options = {}) {
|
|||
process.stdout.write('KTX local embeddings runtime smoke verified\n');
|
||||
} finally {
|
||||
if (daemonStarted) {
|
||||
await run('pnpm', ['exec', 'ktx', 'dev', 'runtime', 'stop'], {
|
||||
await run('pnpm', ['exec', 'ktx', 'admin', 'runtime', 'stop'], {
|
||||
cwd: installDir,
|
||||
env: smokeEnv,
|
||||
timeoutMs: 60_000,
|
||||
|
|
|
|||
|
|
@ -89,23 +89,23 @@ describe('localEmbeddingsSmokeCommands', () => {
|
|||
|
||||
assert.deepEqual(commands.map((command) => command.label), [
|
||||
'ktx public package version',
|
||||
'ktx dev runtime status missing',
|
||||
'ktx dev runtime install local embeddings',
|
||||
'ktx dev runtime status local embeddings ready',
|
||||
'ktx dev runtime start local embeddings',
|
||||
'ktx admin runtime status missing',
|
||||
'ktx admin runtime install local embeddings',
|
||||
'ktx admin runtime status local embeddings ready',
|
||||
'ktx admin runtime start local embeddings',
|
||||
'ktx setup local embeddings',
|
||||
'ktx dev runtime stop local embeddings',
|
||||
'ktx admin runtime stop local embeddings',
|
||||
]);
|
||||
assert.deepEqual(commands[2], {
|
||||
label: 'ktx dev runtime install local embeddings',
|
||||
label: 'ktx admin runtime install local embeddings',
|
||||
command: 'pnpm',
|
||||
args: ['exec', 'ktx', 'dev', 'runtime', 'install', '--feature', 'local-embeddings', '--yes'],
|
||||
args: ['exec', 'ktx', 'admin', 'runtime', 'install', '--feature', 'local-embeddings', '--yes'],
|
||||
timeoutMs: 1_200_000,
|
||||
});
|
||||
assert.deepEqual(commands[4], {
|
||||
label: 'ktx dev runtime start local embeddings',
|
||||
label: 'ktx admin runtime start local embeddings',
|
||||
command: 'pnpm',
|
||||
args: ['exec', 'ktx', 'dev', 'runtime', 'start', '--feature', 'local-embeddings'],
|
||||
args: ['exec', 'ktx', 'admin', 'runtime', 'start', '--feature', 'local-embeddings'],
|
||||
timeoutMs: 300_000,
|
||||
});
|
||||
assert.deepEqual(commands[5].args, [
|
||||
|
|
|
|||
|
|
@ -147,7 +147,7 @@ export async function findPythonArtifacts(pythonDir) {
|
|||
files,
|
||||
RUNTIME_WHEEL_DISTRIBUTION_NAME,
|
||||
'.whl',
|
||||
'kaelio-ktx dev runtime wheel',
|
||||
'kaelio-ktx runtime wheel',
|
||||
pythonDir,
|
||||
RUNTIME_WHEEL_PACKAGE_VERSION,
|
||||
),
|
||||
|
|
@ -606,8 +606,8 @@ try {
|
|||
requireOutput('ktx public package version', version, await installedPackageVersionPattern());
|
||||
|
||||
const runtimeStatusBefore = parseJsonResultWithExitCode(
|
||||
'ktx dev runtime status missing',
|
||||
await run('pnpm', ['exec', 'ktx', 'dev', 'runtime', 'status', '--json']),
|
||||
'ktx admin runtime status missing',
|
||||
await run('pnpm', ['exec', 'ktx', 'admin', 'runtime', 'status', '--json']),
|
||||
1,
|
||||
);
|
||||
assert.equal(runtimeStatusBefore.kind, 'missing');
|
||||
|
|
@ -766,8 +766,8 @@ try {
|
|||
requireOutput('ktx sl query first managed runtime install', slQuery, /orders/);
|
||||
|
||||
const runtimeStatusAfter = parseJsonResult(
|
||||
'ktx dev runtime status ready',
|
||||
await run('pnpm', ['exec', 'ktx', 'dev', 'runtime', 'status', '--json']),
|
||||
'ktx admin runtime status ready',
|
||||
await run('pnpm', ['exec', 'ktx', 'admin', 'runtime', 'status', '--json']),
|
||||
);
|
||||
assert.equal(runtimeStatusAfter.kind, 'ready');
|
||||
assert.deepEqual(runtimeStatusAfter.manifest.features, ['core']);
|
||||
|
|
@ -795,29 +795,29 @@ try {
|
|||
requireOutput('ktx sl query sqlite execute', sqliteSlQuery, /"rows": \\[\\s*\\[\\s*3\\s*\\]\\s*\\]/);
|
||||
process.stdout.write('ktx sl query sqlite execute verified\\n');
|
||||
|
||||
const runtimeDoctor = await run('pnpm', ['exec', 'ktx', 'dev', 'runtime', 'status']);
|
||||
requireSuccess('ktx dev runtime status', runtimeDoctor);
|
||||
requireOutput('ktx dev runtime status', runtimeDoctor, /KTX Python runtime/);
|
||||
requireOutput('ktx dev runtime status', runtimeDoctor, /status: ready/);
|
||||
process.stdout.write('ktx dev runtime status verified\\n');
|
||||
const runtimeDoctor = await run('pnpm', ['exec', 'ktx', 'admin', 'runtime', 'status']);
|
||||
requireSuccess('ktx admin runtime status', runtimeDoctor);
|
||||
requireOutput('ktx admin runtime status', runtimeDoctor, /KTX Python runtime/);
|
||||
requireOutput('ktx admin runtime status', runtimeDoctor, /status: ready/);
|
||||
process.stdout.write('ktx admin runtime status verified\\n');
|
||||
|
||||
const runtimeStart = await run('pnpm', ['exec', 'ktx', 'dev', 'runtime', 'start']);
|
||||
requireSuccess('ktx dev runtime start', runtimeStart);
|
||||
const runtimeStart = await run('pnpm', ['exec', 'ktx', 'admin', 'runtime', 'start']);
|
||||
requireSuccess('ktx admin runtime start', runtimeStart);
|
||||
daemonStarted = true;
|
||||
requireOutput('ktx dev runtime start', runtimeStart, /Started KTX Python daemon/);
|
||||
requireOutput('ktx dev runtime start', runtimeStart, /url: http:\\/\\/127\\.0\\.0\\.1:\\d+/);
|
||||
requireOutput('ktx dev runtime start', runtimeStart, /features: core/);
|
||||
requireOutput('ktx admin runtime start', runtimeStart, /Started KTX Python daemon/);
|
||||
requireOutput('ktx admin runtime start', runtimeStart, /url: http:\\/\\/127\\.0\\.0\\.1:\\d+/);
|
||||
requireOutput('ktx admin runtime start', runtimeStart, /features: core/);
|
||||
|
||||
const runtimeStartReuse = await run('pnpm', ['exec', 'ktx', 'dev', 'runtime', 'start']);
|
||||
requireSuccess('ktx dev runtime start reuse', runtimeStartReuse);
|
||||
requireOutput('ktx dev runtime start reuse', runtimeStartReuse, /Using existing KTX Python daemon/);
|
||||
requireOutput('ktx dev runtime start reuse', runtimeStartReuse, /features: core/);
|
||||
const runtimeStartReuse = await run('pnpm', ['exec', 'ktx', 'admin', 'runtime', 'start']);
|
||||
requireSuccess('ktx admin runtime start reuse', runtimeStartReuse);
|
||||
requireOutput('ktx admin runtime start reuse', runtimeStartReuse, /Using existing KTX Python daemon/);
|
||||
requireOutput('ktx admin runtime start reuse', runtimeStartReuse, /features: core/);
|
||||
|
||||
const runtimeStop = await run('pnpm', ['exec', 'ktx', 'dev', 'runtime', 'stop']);
|
||||
requireSuccess('ktx dev runtime stop', runtimeStop);
|
||||
const runtimeStop = await run('pnpm', ['exec', 'ktx', 'admin', 'runtime', 'stop']);
|
||||
requireSuccess('ktx admin runtime stop', runtimeStop);
|
||||
daemonStarted = false;
|
||||
requireOutput('ktx dev runtime stop', runtimeStop, /Stopped KTX Python daemon/);
|
||||
process.stdout.write('ktx dev runtime daemon lifecycle verified\\n');
|
||||
requireOutput('ktx admin runtime stop', runtimeStop, /Stopped KTX Python daemon/);
|
||||
process.stdout.write('ktx admin runtime daemon lifecycle verified\\n');
|
||||
|
||||
const structuralScan = await run('pnpm', ['exec', 'ktx', 'ingest', 'warehouse',
|
||||
'--project-dir',
|
||||
|
|
@ -847,7 +847,7 @@ try {
|
|||
process.stdout.write('ktx ingest state verified\\n');
|
||||
} finally {
|
||||
if (daemonStarted) {
|
||||
await run('pnpm', ['exec', 'ktx', 'dev', 'runtime', 'stop']);
|
||||
await run('pnpm', ['exec', 'ktx', 'admin', 'runtime', 'stop']);
|
||||
}
|
||||
if (previousRuntimeRoot === undefined) {
|
||||
delete process.env.KTX_RUNTIME_ROOT;
|
||||
|
|
|
|||
|
|
@ -167,7 +167,7 @@ describe('findPythonArtifacts', () => {
|
|||
it('throws when a required Python artifact is missing', async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), 'ktx-artifacts-test-'));
|
||||
try {
|
||||
await assert.rejects(() => findPythonArtifacts(root), /Missing Python artifact: kaelio-ktx dev runtime wheel/);
|
||||
await assert.rejects(() => findPythonArtifacts(root), /Missing Python artifact: kaelio-ktx runtime wheel/);
|
||||
} finally {
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
|
|
@ -491,20 +491,20 @@ describe('verification snippets', () => {
|
|||
assert.doesNotMatch(source, /run\('python'/);
|
||||
assert.match(source, /KTX_RUNTIME_ROOT/);
|
||||
assert.match(source, /managed-runtime/);
|
||||
assert.match(source, /ktx dev runtime status missing/);
|
||||
assert.match(source, /ktx admin runtime status missing/);
|
||||
assert.match(source, /runtimeStatusBefore\.kind, 'missing'/);
|
||||
assert.ok(source.includes(String.raw`Installing KTX Python runtime \(core\) with uv`));
|
||||
assert.match(source, /KTX Python runtime ready:/);
|
||||
assert.match(source, /ktx dev runtime status ready/);
|
||||
assert.match(source, /ktx admin runtime status ready/);
|
||||
assert.match(source, /runtimeStatusAfter\.kind, 'ready'/);
|
||||
assert.match(source, /runtimeStatusAfter\.manifest\.features/);
|
||||
assert.match(source, /ktx dev runtime status/);
|
||||
assert.match(source, /ktx admin runtime status/);
|
||||
assert.match(source, /status: ready/);
|
||||
assert.match(source, /ktx dev runtime start/);
|
||||
assert.match(source, /ktx dev runtime start reuse/);
|
||||
assert.match(source, /ktx admin runtime start/);
|
||||
assert.match(source, /ktx admin runtime start reuse/);
|
||||
assert.match(source, /Using existing KTX Python daemon/);
|
||||
assert.match(source, /ktx dev runtime stop/);
|
||||
assert.doesNotMatch(source, /ktx dev runtime prune/);
|
||||
assert.match(source, /ktx admin runtime stop/);
|
||||
assert.doesNotMatch(source, /ktx admin runtime prune/);
|
||||
assert.doesNotMatch(source, /staleRuntimeDir/);
|
||||
assert.match(source, /run\('pnpm', \[\s*'exec',\s*'ktx',\s*'ingest',\s*'warehouse'/);
|
||||
assert.match(source, /'--deep'/);
|
||||
|
|
|
|||
|
|
@ -166,19 +166,10 @@ function createReleaseConfig(env = process.env) {
|
|||
'pnpm run artifacts:check',
|
||||
'pnpm run release:readiness',
|
||||
].join(' && '),
|
||||
},
|
||||
],
|
||||
[
|
||||
'@semantic-release/npm',
|
||||
{
|
||||
pkgRoot: 'dist/public-npm-package',
|
||||
tarballDir: 'dist/artifacts/npm',
|
||||
},
|
||||
],
|
||||
[
|
||||
'@semantic-release/exec',
|
||||
{
|
||||
publishCmd: 'pnpm run release:published-smoke',
|
||||
publishCmd: [
|
||||
`npm publish dist/artifacts/npm/kaelio-ktx-\${nextRelease.version}.tgz --tag ${tag} --access public --provenance`,
|
||||
'pnpm run release:published-smoke',
|
||||
].join(' && '),
|
||||
},
|
||||
],
|
||||
...releaseGitPlugins(kind),
|
||||
|
|
|
|||
|
|
@ -27,20 +27,20 @@ describe('semantic-release config', () => {
|
|||
]);
|
||||
|
||||
const config = createReleaseConfig({ KTX_RELEASE_KIND: 'rc', GITHUB_REF_NAME: 'main' });
|
||||
assert.deepEqual(
|
||||
assert.equal(
|
||||
config.plugins.find((plugin) => Array.isArray(plugin) && plugin[0] === '@semantic-release/npm'),
|
||||
[
|
||||
'@semantic-release/npm',
|
||||
{
|
||||
pkgRoot: 'dist/public-npm-package',
|
||||
tarballDir: 'dist/artifacts/npm',
|
||||
},
|
||||
],
|
||||
undefined,
|
||||
'@semantic-release/npm must not run; the exec publishCmd publishes the pre-built tarball',
|
||||
);
|
||||
assert.match(
|
||||
releaseExecOptions(config).prepareCmd,
|
||||
/update-public-release-version\.mjs "\$\{nextRelease\.version\}" "next"/,
|
||||
);
|
||||
assert.match(
|
||||
releaseExecOptions(config).publishCmd,
|
||||
/^npm publish dist\/artifacts\/npm\/kaelio-ktx-\$\{nextRelease\.version\}\.tgz --tag next --access public --provenance/,
|
||||
);
|
||||
assert.match(releaseExecOptions(config).publishCmd, /pnpm run release:published-smoke/);
|
||||
assert.doesNotMatch(JSON.stringify(config.plugins), /release:npm-publish/);
|
||||
const releaseFilePluginNames = pluginNames(config).filter(
|
||||
(plugin) => plugin === '@semantic-release/changelog' || plugin === '@semantic-release/git',
|
||||
|
|
@ -62,6 +62,10 @@ describe('semantic-release config', () => {
|
|||
releaseExecOptions(config).prepareCmd,
|
||||
/update-public-release-version\.mjs "\$\{nextRelease\.version\}" "latest"/,
|
||||
);
|
||||
assert.match(
|
||||
releaseExecOptions(config).publishCmd,
|
||||
/^npm publish dist\/artifacts\/npm\/kaelio-ktx-\$\{nextRelease\.version\}\.tgz --tag latest --access public --provenance/,
|
||||
);
|
||||
assert.equal(config.plugins.includes('./scripts/semantic-release-version-policy.cjs'), false);
|
||||
});
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue