diff --git a/docs-site/content/agents-setup.md b/docs-site/content/agents-setup.md index e7307c13..24614ea8 100644 --- a/docs-site/content/agents-setup.md +++ b/docs-site/content/agents-setup.md @@ -150,7 +150,7 @@ Do **not** run `--deep` ingest in this flow - that requires LLM time and is out If the user asks for stronger verification that `sentence-transformers` is actually serving (not just that setup said "ok"), do all of: -1. `ktx dev runtime status --json` → expect `"kind": "ready"` and `"features": [..., "local-embeddings"]`. +1. `ktx admin runtime status --json` → expect `"kind": "ready"` and `"features": [..., "local-embeddings"]`. 2. `pgrep -fa ktx-daemon` → expect a process running `ktx-daemon serve-http`. 3. `curl -sS http://127.0.0.1:/health` → expect HTTP 200 with `{"status":"healthy",…}`. 4. `curl -sS -X POST http://127.0.0.1:/embeddings/compute -H 'content-type: application/json' -d '{"text":"hello"}'` → expect `{"embedding": [...384 floats...]}`. diff --git a/docs-site/content/docs/cli-reference/ktx-admin.mdx b/docs-site/content/docs/cli-reference/ktx-admin.mdx new file mode 100644 index 00000000..c7df5461 --- /dev/null +++ b/docs-site/content/docs/cli-reference/ktx-admin.mdx @@ -0,0 +1,121 @@ +--- +title: "ktx admin" +description: "Low-level project initialization, runtime, and index management." +--- + +`ktx admin` contains low-level project initialization, managed Python runtime, +and local index management commands. Context building lives at the root as +[`ktx ingest`](/docs/cli-reference/ktx-ingest). Most users should start with +`ktx setup`; use `ktx admin` when preparing local fixtures, checking the bundled +runtime, rebuilding local indexes, or debugging runtime state. + +## Command signature + +```bash +ktx admin [options] +``` + +## Subcommands + +| Subcommand | Description | +|-----------|-------------| +| `init [directory]` | Initialize a Git-backed KTX project directory for maintenance scripts | +| `schema` | Print a JSON Schema describing `ktx.yaml` | +| `runtime` | Install, start, stop, and inspect the KTX-managed Python runtime | +| `reindex` | Sync local wiki and semantic-layer search indexes from disk | + +## `admin init` + +| Flag | Description | Default | +|------|-------------|---------| +| `--force` | Rewrite `ktx.yaml` and scaffold files in an existing project | `false` | + +## `admin schema` + +`ktx admin schema` does not require a `ktx.yaml` file or a configured project +directory. Use it from any directory to generate editor or agent schema files. + +| Flag | Description | Default | +|------|-------------|---------| +| `--output ` | Write the schema to a file instead of stdout | - | + +## `admin runtime` Subcommands + +| Subcommand | Description | +|-----------|-------------| +| `install` | Install the bundled Python runtime wheel into the managed runtime | +| `start` | Start the KTX-managed Python HTTP daemon | +| `stop` | Stop the KTX-managed Python HTTP daemon | +| `status` | Show managed Python runtime status and readiness checks | + +## `admin runtime` Options + +| Flag | Description | Default | +|------|-------------|---------| +| `--feature ` | Runtime feature level for `install` and `start` (`core` or `local-embeddings`) | `core` | +| `--json` | Print JSON output for `status` | `false` | +| `--yes` | Accepted by `install` for scripted install commands | `false` | +| `--force` | Reinstall for `install`, or restart for `start` | `false` | +| `--all` | Stop all recorded or discoverable KTX daemon processes with `stop` | `false` | + +## Examples + +```bash +ktx admin init +ktx admin init ./my-project +ktx admin init --force + +ktx admin schema +ktx admin schema --output ./ktx.schema.json + +ktx admin runtime install --yes +ktx admin runtime install --feature local-embeddings --yes +ktx admin runtime status +ktx admin runtime start +ktx admin runtime start --feature local-embeddings +ktx admin runtime stop +ktx admin runtime stop --all + +ktx admin reindex +ktx admin reindex --force +ktx admin reindex --output plain +ktx admin reindex --json +``` + +## Output + +Runtime commands print the runtime root, installed features, daemon URL, daemon +pid, and log paths where relevant. `ktx admin runtime status --json` includes the +runtime status plus readiness checks. + +## `admin reindex` + +`ktx admin reindex` syncs local wiki and semantic-layer search indexes from +files on disk into `.ktx/db.sqlite`. The command discovers `wiki/global/`, each +`wiki/user//` directory, and each `semantic-layer//` +directory except `_schema`. + +```bash +ktx admin reindex +ktx admin reindex --force +ktx admin reindex --output plain +ktx admin reindex --json +``` + +By default, KTX compares stored search text with the files on disk. It only +re-embeds changed rows and removes rows for files that no longer exist. With +`--force`, KTX clears each discovered scope first and then rebuilds it. + +When embeddings are not configured, KTX still writes lexical FTS rows and +prints an embeddings warning. If a scope fails, KTX keeps processing the +remaining scopes and exits with code `1` after output is written. If the local +state database cannot open or the configured managed embedding runtime is +missing, KTX prints the error and exits with code `1`. + +## Common errors + +| Error | Cause | Recovery | +|-------|-------|----------| +| Runtime status reports missing pieces | Packages, Python environment, or linked CLI are not ready | Run `pnpm install`, `pnpm run setup:dev`, `uv sync --all-groups`, then `ktx admin runtime status` | +| Runtime daemon does not start | The managed Python runtime is missing or stale | Run `ktx admin runtime install --yes`, then `ktx admin runtime start` | +| Multiple daemon processes remain | Older daemon state files or stray processes exist | Run `ktx admin runtime stop --all`, then start the runtime again | diff --git a/docs-site/content/docs/cli-reference/ktx-dev.mdx b/docs-site/content/docs/cli-reference/ktx-dev.mdx deleted file mode 100644 index efa3d74b..00000000 --- a/docs-site/content/docs/cli-reference/ktx-dev.mdx +++ /dev/null @@ -1,91 +0,0 @@ ---- -title: "ktx dev" -description: "Low-level project initialization and runtime management." ---- - -`ktx dev` contains low-level project initialization and managed Python runtime -commands. Context building lives at the root as -[`ktx ingest`](/docs/cli-reference/ktx-ingest). Most users should start with -`ktx setup`; use `ktx dev` when preparing local fixtures, checking the bundled -runtime, or debugging runtime state. - -## Command signature - -```bash -ktx dev [options] -``` - -## Subcommands - -| Subcommand | Description | -|-----------|-------------| -| `init [directory]` | Initialize a Git-backed KTX project directory for maintenance scripts | -| `schema` | Print a JSON Schema describing `ktx.yaml` | -| `runtime` | Install, start, stop, and inspect the KTX-managed Python runtime | - -## `dev init` - -| Flag | Description | Default | -|------|-------------|---------| -| `--force` | Rewrite `ktx.yaml` and scaffold files in an existing project | `false` | - -## `dev schema` - -`ktx dev schema` does not require a `ktx.yaml` file or a configured project -directory. Use it from any directory to generate editor or agent schema files. - -| Flag | Description | Default | -|------|-------------|---------| -| `--output ` | Write the schema to a file instead of stdout | - | - -## `dev runtime` Subcommands - -| Subcommand | Description | -|-----------|-------------| -| `install` | Install the bundled Python runtime wheel into the managed runtime | -| `start` | Start the KTX-managed Python HTTP daemon | -| `stop` | Stop the KTX-managed Python HTTP daemon | -| `status` | Show managed Python runtime status and readiness checks | - -## `dev runtime` Options - -| Flag | Description | Default | -|------|-------------|---------| -| `--feature ` | Runtime feature level for `install` and `start` (`core` or `local-embeddings`) | `core` | -| `--json` | Print JSON output for `status` | `false` | -| `--yes` | Accepted by `install` for scripted install commands | `false` | -| `--force` | Reinstall for `install`, or restart for `start` | `false` | -| `--all` | Stop all recorded or discoverable KTX daemon processes with `stop` | `false` | - -## Examples - -```bash -ktx dev init -ktx dev init ./my-project -ktx dev init --force - -ktx dev schema -ktx dev schema --output ./ktx.schema.json - -ktx dev runtime install --yes -ktx dev runtime install --feature local-embeddings --yes -ktx dev runtime status -ktx dev runtime start -ktx dev runtime start --feature local-embeddings -ktx dev runtime stop -ktx dev runtime stop --all -``` - -## Output - -Runtime commands print the runtime root, installed features, daemon URL, daemon -pid, and log paths where relevant. `ktx dev runtime status --json` includes the -runtime status plus readiness checks. - -## Common errors - -| Error | Cause | Recovery | -|-------|-------|----------| -| Runtime status reports missing pieces | Packages, Python environment, or linked CLI are not ready | Run `pnpm install`, `pnpm run setup:dev`, `uv sync --all-groups`, then `ktx dev runtime status` | -| Runtime daemon does not start | The managed Python runtime is missing or stale | Run `ktx dev runtime install --yes`, then `ktx dev runtime start` | -| Multiple daemon processes remain | Older daemon state files or stray processes exist | Run `ktx dev runtime stop --all`, then start the runtime again | diff --git a/docs-site/content/docs/cli-reference/ktx-ingest.mdx b/docs-site/content/docs/cli-reference/ktx-ingest.mdx index 49485d10..3708af04 100644 --- a/docs-site/content/docs/cli-reference/ktx-ingest.mdx +++ b/docs-site/content/docs/cli-reference/ktx-ingest.mdx @@ -153,7 +153,7 @@ KTX_INGEST_TRACE_LEVEL=trace ktx ingest metabase | Connection not configured | The connection id is not present in `ktx.yaml` | Add the connection with `ktx setup` or update `ktx.yaml` | | Deep readiness is missing | `--deep` or query history needs model, embedding, and scan-enrichment configuration | Run `ktx setup` or rerun with `--fast` | | Query history is unsupported | The selected database driver does not support query history | Run schema ingest without query-history flags | -| Python runtime is missing | The selected ingest target needs runtime-backed SQL analysis or source parsing | Accept the interactive prompt, rerun with `--yes`, or run the suggested `ktx dev runtime install` command | +| Python runtime is missing | The selected ingest target needs runtime-backed SQL analysis or source parsing | Accept the interactive prompt, rerun with `--yes`, or run the suggested `ktx admin runtime install` command | | No ingest target was selected | No connection id was provided and `--all` was omitted | Run `ktx ingest ` or `ktx ingest --all` | | Source options were ignored | Depth and query-history flags were supplied for a non-database source | Omit database-only flags when ingesting source connections | | Text ingest stops early | `--fail-fast` was used and one item failed | Fix the failed item or rerun without `--fail-fast` to collect all failures | diff --git a/docs-site/content/docs/cli-reference/ktx-setup.mdx b/docs-site/content/docs/cli-reference/ktx-setup.mdx index 6dbe54c2..e98a51e4 100644 --- a/docs-site/content/docs/cli-reference/ktx-setup.mdx +++ b/docs-site/content/docs/cli-reference/ktx-setup.mdx @@ -231,7 +231,7 @@ Use `ktx status` for repeatable readiness checks after setup exits. | Setup resumes an unexpected project | `KTX_PROJECT_DIR` or nearest `ktx.yaml` points to another directory | Pass `--project-dir ` explicitly | | Setup cannot run in CI | Required values are missing and `--no-input` disables prompts | Provide the relevant automation flags or create a fixture `ktx.yaml` | | Provider health check fails | Provider key, model id, Vertex project, or Vertex location is invalid | Fix the `env:` or `file:` reference and rerun setup | -| Python runtime is missing | The selected setup needs runtime-backed agent, query-history, Looker, or local embedding features | Accept the interactive prompt, rerun with `--yes`, or run the suggested `ktx dev runtime install` command | +| Python runtime is missing | The selected setup needs runtime-backed agent, query-history, Looker, or local embedding features | Accept the interactive prompt, rerun with `--yes`, or run the suggested `ktx admin runtime install` command | | `--enable-query-history` is rejected | The selected database driver does not support query history | Use Postgres, BigQuery, or Snowflake, or rerun without query-history flags | | Source setup rejects location flags | Both `--source-path` and `--source-git-url` were supplied | Choose the local path or the Git URL, not both | | Agent integration missing | Setup skipped the agents step | Run `ktx setup --agents --target ` | diff --git a/docs-site/content/docs/cli-reference/ktx-sl.mdx b/docs-site/content/docs/cli-reference/ktx-sl.mdx index f395a170..85233cb6 100644 --- a/docs-site/content/docs/cli-reference/ktx-sl.mdx +++ b/docs-site/content/docs/cli-reference/ktx-sl.mdx @@ -165,4 +165,4 @@ ranking score rather than a percentage. | Validation fails | YAML references missing columns, invalid joins, or invalid SQL expressions | Fix the source YAML and rerun `ktx sl validate` | | Query compile fails | Measure, dimension, filter, or segment name is invalid | Search sources with `ktx sl search`, inspect the source YAML in your project files, then retry using declared fields | | Execution returns too many rows | `--max-rows` is missing or too high | Add `--max-rows` with a bounded value before executing | -| Runtime install is blocked | Query execution needs the managed Python runtime and prompts are disabled | Run `ktx dev runtime install --feature core --yes`, or rerun `ktx sl query --yes` | +| Runtime install is blocked | Query execution needs the managed Python runtime and prompts are disabled | Run `ktx admin runtime install --feature core --yes`, or rerun `ktx sl query --yes` | diff --git a/docs-site/content/docs/cli-reference/ktx.mdx b/docs-site/content/docs/cli-reference/ktx.mdx index 937d2529..d47b9867 100644 --- a/docs-site/content/docs/cli-reference/ktx.mdx +++ b/docs-site/content/docs/cli-reference/ktx.mdx @@ -48,7 +48,7 @@ ktx stop status logs - dev + admin init [directory] schema runtime @@ -56,6 +56,7 @@ ktx start stop status + reindex ``` The public context-build entrypoint is `ktx ingest [connectionId]` or diff --git a/docs-site/content/docs/cli-reference/meta.json b/docs-site/content/docs/cli-reference/meta.json index 6385c7d4..49eb8ba7 100644 --- a/docs-site/content/docs/cli-reference/meta.json +++ b/docs-site/content/docs/cli-reference/meta.json @@ -11,6 +11,6 @@ "ktx-wiki", "ktx-status", "ktx-mcp", - "ktx-dev" + "ktx-admin" ] } diff --git a/docs-site/content/docs/getting-started/quickstart.mdx b/docs-site/content/docs/getting-started/quickstart.mdx index d0566ec9..14fe476f 100644 --- a/docs-site/content/docs/getting-started/quickstart.mdx +++ b/docs-site/content/docs/getting-started/quickstart.mdx @@ -126,8 +126,8 @@ If you choose local `sentence-transformers` embeddings, KTX uses the managed Python runtime. To prepare it before setup, run: ```bash -ktx dev runtime install --feature local-embeddings --yes -ktx dev runtime start --feature local-embeddings +ktx admin runtime install --feature local-embeddings --yes +ktx admin runtime start --feature local-embeddings ``` During the database step, setup tests the saved connection and builds initial diff --git a/examples/package-artifacts/README.md b/examples/package-artifacts/README.md index 22ecaf92..ed72f13f 100644 --- a/examples/package-artifacts/README.md +++ b/examples/package-artifacts/README.md @@ -12,8 +12,8 @@ imports the package entry point, and runs installed `ktx` commands against a generated local project. The managed Python runtime smoke requires `uv` on `PATH`, isolates -`KTX_RUNTIME_ROOT`, verifies `ktx dev runtime status`, runs `ktx sl query --yes` to -install the core runtime from the bundled wheel, checks `ktx dev runtime status`, +`KTX_RUNTIME_ROOT`, verifies `ktx admin runtime status`, runs `ktx sl query --yes` to +install the core runtime from the bundled wheel, checks `ktx admin runtime status`, starts and reuses the managed daemon, and stops it. The artifact manifest contains the public `@kaelio/ktx` npm tarball and the diff --git a/packages/cli/src/admin-reindex.test.ts b/packages/cli/src/admin-reindex.test.ts new file mode 100644 index 00000000..eb75c651 --- /dev/null +++ b/packages/cli/src/admin-reindex.test.ts @@ -0,0 +1,145 @@ +import type { ReindexSummary } from '@ktx/context/index-sync'; +import { describe, expect, it, vi } from 'vitest'; +import { renderReindexJson, renderReindexPlain, reindexHasErrors } from './admin-reindex.js'; +import { runKtxCli } from './index.js'; + +function makeIo(options: { stdoutIsTTY?: boolean } = {}) { + let stdout = ''; + let stderr = ''; + return { + io: { + stdout: { + isTTY: options.stdoutIsTTY, + write: (chunk: string) => { + stdout += chunk; + }, + }, + stderr: { + write: (chunk: string) => { + stderr += chunk; + }, + }, + }, + stdout: () => stdout, + stderr: () => stderr, + }; +} + +function summary(overrides: Partial = {}): ReindexSummary { + return { + scopes: [ + { + kind: 'wiki', + label: 'global', + scope: 'global', + scopeId: null, + scanned: 42, + updated: 3, + deleted: 1, + embeddingsRecomputed: 3, + embeddingsFailed: 0, + durationMs: 412, + }, + { + kind: 'sl', + label: 'warehouse', + connectionId: 'warehouse', + scanned: 18, + updated: 2, + deleted: 0, + embeddingsRecomputed: 2, + embeddingsFailed: 0, + durationMs: 287, + }, + ], + totals: { scanned: 60, updated: 5, deleted: 1, embeddingsRecomputed: 5, embeddingsFailed: 0 }, + dbPath: '.ktx/db.sqlite', + force: false, + embeddingsAvailable: true, + durationMs: 1234, + ...overrides, + }; +} + +describe('admin reindex renderers', () => { + it('renders plain scope lines to stderr and summary to stdout', () => { + const io = makeIo(); + + renderReindexPlain(summary(), io.io); + + expect(io.stderr()).toContain('wiki/global\tscanned=42\tupdated=3\tdeleted=1\tembeddings=3\tduration_ms=412\n'); + expect(io.stderr()).toContain('sl/warehouse\tscanned=18\tupdated=2\tdeleted=0\tembeddings=2\tduration_ms=287\n'); + expect(io.stdout()).toBe('reindex\tscopes=2\tscanned=60\tupdated=5\tdeleted=1\tembeddings=5\tduration_ms=1234\n'); + }); + + it('renders rebuilt labels in plain force mode', () => { + const io = makeIo(); + + renderReindexPlain(summary({ force: true }), io.io); + + expect(io.stderr()).toContain('rebuilt=3'); + expect(io.stdout()).toContain('rebuilt=5'); + expect(io.stdout()).not.toContain('updated=5'); + }); + + it('renders json envelope to stdout only', () => { + const io = makeIo(); + + renderReindexJson(summary(), io.io); + + expect(JSON.parse(io.stdout())).toMatchObject({ + kind: 'reindex', + data: { totals: { scanned: 60, updated: 5 } }, + meta: { command: 'admin reindex' }, + }); + expect(io.stderr()).toBe(''); + }); + + it('detects per-scope errors', () => { + expect( + reindexHasErrors( + summary({ + scopes: [{ ...summary().scopes[0]!, error: 'provider failed' }], + }), + ), + ).toBe(true); + }); +}); + +describe('admin reindex Commander routing', () => { + it('routes flags to the injectable reindex runner', async () => { + const { mkdir, mkdtemp, rm, writeFile } = await import('node:fs/promises'); + const { tmpdir } = await import('node:os'); + const { join } = await import('node:path'); + const tempDir = await mkdtemp(join(tmpdir(), 'ktx-admin-reindex-cli-')); + const projectDir = join(tempDir, 'project'); + const io = makeIo(); + const adminReindex = vi.fn(async () => 0); + + try { + await mkdir(projectDir, { recursive: true }); + await writeFile(join(projectDir, 'ktx.yaml'), '{}\n', 'utf-8'); + + await expect( + runKtxCli( + ['--project-dir', projectDir, 'admin', 'reindex', '--force', '--json', '--output', 'plain'], + io.io, + { adminReindex }, + ), + ).resolves.toBe(0); + } finally { + await rm(tempDir, { recursive: true, force: true }); + } + + expect(adminReindex).toHaveBeenCalledWith( + { + projectDir, + force: true, + json: true, + output: 'plain', + cliVersion: '0.1.0-rc.1', + }, + io.io, + ); + }); +}); diff --git a/packages/cli/src/admin-reindex.ts b/packages/cli/src/admin-reindex.ts new file mode 100644 index 00000000..8518fc2c --- /dev/null +++ b/packages/cli/src/admin-reindex.ts @@ -0,0 +1,210 @@ +import { + createLocalKtxEmbeddingProviderFromConfig, + KtxIngestEmbeddingPortAdapter, + MANAGED_SENTENCE_TRANSFORMERS_BASE_URL, + type KtxEmbeddingPort, +} from '@ktx/context'; +import { reindexLocalIndexes, type ReindexScopeResult, type ReindexSummary } from '@ktx/context/index-sync'; +import { loadKtxProject, type KtxLocalProject } from '@ktx/context/project'; +import { Option, type Command } from '@commander-js/extra-typings'; +import { cancel, intro, log, note, outro } from '@clack/prompts'; +import type { KtxCliCommandContext } from './cli-program.js'; +import type { KtxCliIo } from './cli-runtime.js'; +import { resolveOutputMode } from './io/mode.js'; +import { green, red, SYMBOLS } from './io/symbols.js'; +import { ensureManagedLocalEmbeddingsDaemon } from './managed-local-embeddings.js'; + +export interface KtxAdminReindexArgs { + projectDir: string; + force: boolean; + output?: 'pretty' | 'plain' | 'json'; + json?: boolean; + cliVersion: string; +} + +export function registerAdminReindexCommand(admin: Command, context: KtxCliCommandContext): void { + admin + .command('reindex') + .description('Sync local wiki and semantic-layer search indexes from disk') + .option('--force', 'Clear each discovered scope before rebuilding it', false) + .option('--json', 'Shortcut for --output=json (overrides --output)', false) + .addOption( + new Option('--output ', 'Output mode: pretty, plain, or json').choices(['pretty', 'plain', 'json']), + ) + .action(async (options: { force?: boolean; json?: boolean; output?: 'pretty' | 'plain' | 'json' }, command) => { + const runner = context.deps.adminReindex ?? runKtxAdminReindex; + const { resolveCommandProjectDir } = await import('./cli-program.js'); + context.setExitCode( + await runner( + { + projectDir: resolveCommandProjectDir(command), + force: options.force === true, + json: options.json === true, + output: options.output, + cliVersion: context.packageInfo.version, + }, + context.io, + ), + ); + }); +} + +async function resolveReindexEmbeddingService( + project: KtxLocalProject, + args: KtxAdminReindexArgs, + io: KtxCliIo, +): Promise { + const config = project.config.ingest.embeddings; + if (config.backend === 'none') { + return null; + } + + if ( + config.backend === 'sentence-transformers' && + config.sentenceTransformers?.base_url === MANAGED_SENTENCE_TRANSFORMERS_BASE_URL + ) { + const daemon = await ensureManagedLocalEmbeddingsDaemon({ + cliVersion: args.cliVersion, + projectDir: project.projectDir, + installPolicy: 'never', + io, + }); + const provider = createLocalKtxEmbeddingProviderFromConfig(config, { env: { ...process.env, ...daemon.env } }); + return provider ? new KtxIngestEmbeddingPortAdapter(provider) : null; + } + + const provider = createLocalKtxEmbeddingProviderFromConfig(config); + return provider ? new KtxIngestEmbeddingPortAdapter(provider) : null; +} + +function scopeKey(scope: ReindexScopeResult): string { + if (scope.kind === 'wiki') { + return scope.scope === 'user' ? `wiki/user/${scope.scopeId ?? 'local'}` : 'wiki/global'; + } + return `sl/${scope.connectionId ?? scope.label}`; +} + +function quotePlainValue(value: string): string { + return `"${value.replaceAll('\\', '\\\\').replaceAll('"', '\\"')}"`; +} + +export function reindexHasErrors(summary: ReindexSummary): boolean { + return summary.scopes.some((scope) => scope.error); +} + +export function renderReindexPlain(summary: ReindexSummary, io: KtxCliIo): void { + const updateKey = summary.force ? 'rebuilt' : 'updated'; + for (const scope of summary.scopes) { + const cells = [ + scopeKey(scope), + `scanned=${scope.scanned}`, + `${updateKey}=${scope.updated}`, + `deleted=${scope.deleted}`, + `embeddings=${summary.embeddingsAvailable ? String(scope.embeddingsRecomputed) : '-'}`, + `duration_ms=${scope.durationMs}`, + ...(scope.error ? [`error=${quotePlainValue(scope.error)}`] : []), + ]; + io.stderr.write(`${cells.join('\t')}\n`); + } + const failed = summary.scopes.filter((scope) => scope.error).length; + io.stdout.write( + [ + 'reindex', + `scopes=${summary.scopes.length}`, + `scanned=${summary.totals.scanned}`, + `${updateKey}=${summary.totals.updated}`, + `deleted=${summary.totals.deleted}`, + `embeddings=${summary.embeddingsAvailable ? String(summary.totals.embeddingsRecomputed) : '-'}`, + `duration_ms=${summary.durationMs}`, + ...(failed > 0 ? [`failed=${failed}`] : []), + ].join('\t') + '\n', + ); +} + +export function renderReindexJson(summary: ReindexSummary, io: KtxCliIo): void { + io.stdout.write(`${JSON.stringify({ kind: 'reindex', data: summary, meta: { command: 'admin reindex' } }, null, 2)}\n`); +} + +function noun(scope: ReindexScopeResult): string { + return scope.kind === 'wiki' ? 'pages' : 'sources'; +} + +function formatScopeLine(scope: ReindexScopeResult, force: boolean, embeddingsAvailable: boolean): string { + if (scope.error) { + return `${scope.kind === 'wiki' ? 'Wiki' : 'SL'}: ${scope.label} ${SYMBOLS.emDash} failed: ${scope.error}`; + } + const changedLabel = force ? 'rebuilt' : 'updated'; + const parts = [`${scope.scanned} ${noun(scope)}`]; + if (scope.updated > 0) { + parts.push(`${scope.updated} ${changedLabel}`); + } else { + parts.push('unchanged'); + } + if (!force && scope.deleted > 0) { + parts.push(`${scope.deleted} deleted`); + } + if (embeddingsAvailable) { + parts.push(`${scope.embeddingsRecomputed} embeddings recomputed`); + } + parts.push(`${scope.durationMs}ms`); + return `${scope.kind === 'wiki' ? 'Wiki' : 'SL'}: ${scope.label} ${SYMBOLS.emDash} ${parts.join(` ${SYMBOLS.middot} `)}`; +} + +function renderReindexPretty(summary: ReindexSummary, io: KtxCliIo): void { + intro(summary.force ? 'ktx admin reindex --force' : 'ktx admin reindex'); + if (!summary.embeddingsAvailable) { + log.warn(`Embeddings: not configured ${SYMBOLS.emDash} indexing lexical only`); + } + for (const scope of summary.scopes) { + const line = formatScopeLine(scope, summary.force, summary.embeddingsAvailable); + if (scope.error) { + log.error(red(line)); + } else { + log.success(green(line)); + } + } + const failed = summary.scopes.filter((scope) => scope.error).length; + note( + [ + `scopes ${summary.scopes.length}`, + `scanned ${summary.totals.scanned}`, + `${summary.force ? 'rebuilt' : 'updated'} ${summary.totals.updated}`, + `deleted ${summary.totals.deleted}`, + `embeddings ${summary.embeddingsAvailable ? summary.totals.embeddingsRecomputed : SYMBOLS.emDash}`, + `index ${summary.dbPath}`, + ...(failed > 0 ? [`failed ${failed}`] : []), + ].join('\n'), + 'Summary', + ); + if (failed > 0) { + cancel(`reindex completed with ${failed} error${failed === 1 ? '' : 's'}`); + } else { + outro(`Done in ${(summary.durationMs / 1000).toFixed(1)}s`); + } + void io; +} + +async function runKtxAdminReindex(args: KtxAdminReindexArgs, io: KtxCliIo = process): Promise { + try { + const project = await loadKtxProject({ projectDir: args.projectDir }); + const embeddingService = await resolveReindexEmbeddingService(project, args, io); + const summary = await reindexLocalIndexes(project, { force: args.force, embeddingService }); + const mode = resolveOutputMode({ explicit: args.output, json: args.json, io }); + + if (!summary.embeddingsAvailable && mode === 'plain') { + io.stderr.write(`Embeddings: not configured ${SYMBOLS.emDash} indexing lexical only\n`); + } + + if (mode === 'json') { + renderReindexJson(summary, io); + } else if (mode === 'plain') { + renderReindexPlain(summary, io); + } else { + renderReindexPretty(summary, io); + } + return reindexHasErrors(summary) ? 1 : 0; + } catch (error) { + io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + return 1; + } +} diff --git a/packages/cli/src/dev.test.ts b/packages/cli/src/admin.test.ts similarity index 75% rename from packages/cli/src/dev.test.ts rename to packages/cli/src/admin.test.ts index 95adb20c..15f4179e 100644 --- a/packages/cli/src/dev.test.ts +++ b/packages/cli/src/admin.test.ts @@ -22,14 +22,14 @@ function makeIo() { }; } -describe('dev Commander tree', () => { - it('prints visible dev help with only supported low-level command groups', async () => { +describe('admin Commander tree', () => { + it('prints visible admin help with supported low-level command groups', async () => { const testIo = makeIo(); - await expect(runKtxCli(['dev', '--help'], testIo.io)).resolves.toBe(0); + await expect(runKtxCli(['admin', '--help'], testIo.io)).resolves.toBe(0); - expect(testIo.stdout()).toContain('Usage: ktx dev [options] [command]'); - for (const command of ['init', 'runtime']) { + expect(testIo.stdout()).toContain('Usage: ktx admin [options] [command]'); + for (const command of ['init', 'runtime', 'reindex']) { expect(testIo.stdout()).toContain(command); } for (const removed of [ @@ -52,27 +52,35 @@ describe('dev Commander tree', () => { expect(testIo.stderr()).toBe(''); }); - it('lists dev in root command rows', async () => { + it('lists admin in root command rows', async () => { const testIo = makeIo(); await expect(runKtxCli(['--help'], testIo.io)).resolves.toBe(0); expect(testIo.stdout()).not.toContain('Advanced:'); - expect(testIo.stdout()).toContain('dev'); - expect(testIo.stdout()).toMatch(/Low-level project initialization and runtime\s+management/); + expect(testIo.stdout()).toContain('admin'); + expect(testIo.stdout()).toMatch(/Low-level project initialization,\s+runtime,\s+and index management/); expect(testIo.stderr()).toBe(''); }); - it('keeps project scaffolding under dev init', async () => { + it('does not keep a dev alias', async () => { + const testIo = makeIo(); + + await expect(runKtxCli(['dev', '--help'], testIo.io)).resolves.toBe(1); + + expect(testIo.stderr()).toContain("unknown command 'dev'"); + }); + + it('keeps project scaffolding under admin init', async () => { const { mkdtemp, readFile, rm } = await import('node:fs/promises'); const { tmpdir } = await import('node:os'); const { join } = await import('node:path'); - const tempDir = await mkdtemp(join(tmpdir(), 'ktx-dev-init-')); + const tempDir = await mkdtemp(join(tmpdir(), 'ktx-admin-init-')); const projectDir = join(tempDir, 'warehouse'); const testIo = makeIo(); try { - await expect(runKtxCli(['dev', 'init', projectDir], testIo.io)).resolves.toBe(0); + await expect(runKtxCli(['admin', 'init', projectDir], testIo.io)).resolves.toBe(0); expect(testIo.stdout()).toContain(`Initialized KTX project at ${projectDir}`); await expect(readFile(join(projectDir, 'ktx.yaml'), 'utf-8')).resolves.not.toContain('project:'); @@ -82,17 +90,17 @@ describe('dev Commander tree', () => { } }); - it('uses global project-dir for dev init when the positional directory is omitted', async () => { + it('uses global project-dir for admin init when the positional directory is omitted', async () => { const { mkdtemp, rm } = await import('node:fs/promises'); const { tmpdir } = await import('node:os'); const { join } = await import('node:path'); - const tempDir = await mkdtemp(join(tmpdir(), 'ktx-dev-init-global-')); + const tempDir = await mkdtemp(join(tmpdir(), 'ktx-admin-init-global-')); const projectDir = join(tempDir, 'global-init'); const testIo = makeIo(); try { await expect( - runKtxCli(['--project-dir', projectDir, 'dev', 'init'], testIo.io), + runKtxCli(['--project-dir', projectDir, 'admin', 'init'], testIo.io), ).resolves.toBe(0); expect(testIo.stdout()).toContain(`Initialized KTX project at ${projectDir}`); @@ -106,7 +114,7 @@ describe('dev Commander tree', () => { const { mkdtemp, rm } = await import('node:fs/promises'); const { tmpdir } = await import('node:os'); const { join } = await import('node:path'); - const tempDir = await mkdtemp(join(tmpdir(), 'ktx-dev-schema-')); + const tempDir = await mkdtemp(join(tmpdir(), 'ktx-admin-schema-')); const missingProjectDir = join(tempDir, 'missing-project'); const originalProjectDir = process.env.KTX_PROJECT_DIR; const testIo = makeIo(); @@ -114,7 +122,7 @@ describe('dev Commander tree', () => { try { process.env.KTX_PROJECT_DIR = missingProjectDir; - await expect(runKtxCli(['dev', 'schema'], testIo.io)).resolves.toBe(0); + await expect(runKtxCli(['admin', 'schema'], testIo.io)).resolves.toBe(0); expect(JSON.parse(testIo.stdout())).toMatchObject({ title: 'ktx.yaml', @@ -131,19 +139,19 @@ describe('dev Commander tree', () => { } }); - it('rejects removed dev command groups', async () => { + it('rejects removed admin command groups', async () => { for (const argv of [ - ['dev', 'doctor', 'setup'], - ['dev', 'runtime', 'doctor'], - ['dev', 'runtime', 'prune', '--dry-run'], - ['dev', 'scan', 'warehouse'], - ['dev', 'ingest', 'run'], - ['dev', 'mapping', 'list'], - ['dev', 'completion', 'zsh'], - ['dev', '__complete', '--shell', 'zsh', '--position', '2', '--', 'ktx', ''], - ['dev', 'knowledge', 'list'], - ['dev', 'model', 'list'], - ['dev', 'artifacts'], + ['admin', 'doctor', 'setup'], + ['admin', 'runtime', 'doctor'], + ['admin', 'runtime', 'prune', '--dry-run'], + ['admin', 'scan', 'warehouse'], + ['admin', 'ingest', 'run'], + ['admin', 'mapping', 'list'], + ['admin', 'completion', 'zsh'], + ['admin', '__complete', '--shell', 'zsh', '--position', '2', '--', 'ktx', ''], + ['admin', 'knowledge', 'list'], + ['admin', 'model', 'list'], + ['admin', 'artifacts'], ]) { const testIo = makeIo(); @@ -155,8 +163,8 @@ describe('dev Commander tree', () => { it.each([ { - argv: ['dev', 'runtime', '--help'], - expected: ['Usage: ktx dev runtime', 'install', 'start', 'stop', 'status'], + argv: ['admin', 'runtime', '--help'], + expected: ['Usage: ktx admin runtime', 'install', 'start', 'stop', 'status'], }, ])('prints generated nested help for $argv', async ({ argv, expected }) => { const io = makeIo(); @@ -167,7 +175,7 @@ describe('dev Commander tree', () => { for (const text of expected) { expect(io.stdout()).toContain(text); } - if (argv.join(' ') === 'dev runtime --help') { + if (argv.join(' ') === 'admin runtime --help') { expect(io.stdout()).not.toContain('prune'); expect(io.stdout()).not.toContain('doctor'); } diff --git a/packages/cli/src/dev.ts b/packages/cli/src/admin.ts similarity index 76% rename from packages/cli/src/dev.ts rename to packages/cli/src/admin.ts index 12ad6f46..af23c192 100644 --- a/packages/cli/src/dev.ts +++ b/packages/cli/src/admin.ts @@ -1,27 +1,28 @@ import { resolve } from 'node:path'; import type { Command } from '@commander-js/extra-typings'; import { type CommandWithGlobalOptions, type KtxCliCommandContext, resolveCommandProjectDir } from './cli-program.js'; +import { registerAdminReindexCommand } from './admin-reindex.js'; import { registerRuntimeCommands } from './commands/runtime-commands.js'; import { profileMark } from './startup-profile.js'; -profileMark('module:dev'); +profileMark('module:admin'); -export function registerDevCommands(program: Command, context: KtxCliCommandContext): void { - const dev = program - .command('dev') - .description('Low-level project initialization and runtime management') +export function registerAdminCommands(program: Command, context: KtxCliCommandContext): void { + const admin = program + .command('admin') + .description('Low-level project initialization, runtime, and index management') .showHelpAfterError(); - dev.hook('preAction', (_thisCommand, actionCommand) => { - context.writeDebug?.('dev', actionCommand); + admin.hook('preAction', (_thisCommand, actionCommand) => { + context.writeDebug?.('admin', actionCommand); }); - dev.action(() => { - dev.outputHelp(); + admin.action(() => { + admin.outputHelp(); context.setExitCode(0); }); - dev + admin .command('init') .description('Initialize a Git-backed KTX project directory for maintenance scripts') .argument('[directory]', 'Project directory') @@ -44,7 +45,7 @@ export function registerDevCommands(program: Command, context: KtxCliCommandCont }, ); - dev + admin .command('schema') .description('Print a JSON Schema describing ktx.yaml (for editors and LLM agents)') .option('--output ', 'Write the schema to a file instead of stdout') @@ -62,5 +63,6 @@ export function registerDevCommands(program: Command, context: KtxCliCommandCont context.setExitCode(0); }); - registerRuntimeCommands(dev, context); + registerRuntimeCommands(admin, context); + registerAdminReindexCommand(admin, context); } diff --git a/packages/cli/src/cli-program.test.ts b/packages/cli/src/cli-program.test.ts index f0ac9595..2e5333c9 100644 --- a/packages/cli/src/cli-program.test.ts +++ b/packages/cli/src/cli-program.test.ts @@ -31,7 +31,7 @@ describe('buildKtxProgram', () => { expect(program.name()).toBe('ktx'); const topLevel = program.commands.map((command) => command.name()).sort(); - for (const expected of ['setup', 'connection', 'ingest', 'sl', 'dev']) { + for (const expected of ['setup', 'connection', 'ingest', 'sl', 'admin']) { expect(topLevel).toContain(expected); } }); diff --git a/packages/cli/src/cli-program.ts b/packages/cli/src/cli-program.ts index 34b13854..84f740f5 100644 --- a/packages/cli/src/cli-program.ts +++ b/packages/cli/src/cli-program.ts @@ -10,7 +10,7 @@ import { registerSetupCommands } from './commands/setup-commands.js'; import { registerSlCommands } from './commands/sl-commands.js'; import { registerSqlCommands } from './commands/sql-commands.js'; import { registerStatusCommands } from './commands/status-commands.js'; -import { registerDevCommands } from './dev.js'; +import { registerAdminCommands } from './admin.js'; import { renderMissingProjectMessage } from './doctor.js'; import { findNearestKtxProjectDir, resolveKtxProjectDir } from './project-resolver.js'; import { profileMark, profileSpan } from './startup-profile.js'; @@ -58,8 +58,8 @@ type CommandPathNode = CommandWithGlobalOptions & { }; const PROJECT_AWARE_ROOT_COMMANDS = new Set(['setup', 'connection', 'ingest', 'wiki', 'sl', 'sql', 'status', 'mcp']); -const PROJECT_INDEPENDENT_DEV_COMMANDS = new Set(['runtime', 'schema']); -const COMMANDS_THAT_CREATE_PROJECT = new Set(['setup', 'ktx dev init']); +const PROJECT_INDEPENDENT_ADMIN_COMMANDS = new Set(['runtime', 'schema']); +const COMMANDS_THAT_CREATE_PROJECT = new Set(['setup', 'ktx admin init']); const COMMANDS_WITH_OWN_MISSING_PROJECT_HANDLING = new Set(['status']); const GLOBAL_OPTIONS_WITH_VALUE = new Set(['--project-dir']); const GLOBAL_OPTIONS_WITHOUT_VALUE = new Set(['--debug', '--help', '-h', '--version', '-v']); @@ -172,15 +172,15 @@ function isProjectAwareCommand(path: string[]): boolean { } const rootCommand = path[1]; - if (rootCommand === 'dev') { - return path[2] !== undefined && !PROJECT_INDEPENDENT_DEV_COMMANDS.has(path[2]); + if (rootCommand === 'admin') { + return path[2] !== undefined && !PROJECT_INDEPENDENT_ADMIN_COMMANDS.has(path[2]); } return rootCommand !== undefined && PROJECT_AWARE_ROOT_COMMANDS.has(rootCommand); } function shouldSuppressProjectDirLine(path: string[], options: Record): boolean { const commandPathKey = path.join(' '); - if (commandPathKey === 'ktx dev init') { + if (commandPathKey === 'ktx admin init') { return true; } @@ -421,7 +421,7 @@ export function buildKtxProgram(options: BuildKtxProgramOptions): Command { registerSqlCommands(program, context); registerStatusCommands(program, context); registerMcpCommands(program, context); - registerDevCommands(program, context); + registerAdminCommands(program, context); return program; } diff --git a/packages/cli/src/cli-runtime.ts b/packages/cli/src/cli-runtime.ts index b8bc636d..a2d4765c 100644 --- a/packages/cli/src/cli-runtime.ts +++ b/packages/cli/src/cli-runtime.ts @@ -1,6 +1,7 @@ import { createRequire } from 'node:module'; import type { KtxConnectionArgs } from './connection.js'; +import type { KtxAdminReindexArgs } from './admin-reindex.js'; import type { KtxDoctorArgs } from './doctor.js'; import type { KtxKnowledgeArgs } from './knowledge.js'; import type { KtxPublicIngestArgs } from './public-ingest.js'; @@ -30,6 +31,7 @@ export interface KtxCliIo { } export interface KtxCliDeps { + adminReindex?: (args: KtxAdminReindexArgs, io: KtxCliIo) => Promise; setup?: (args: KtxSetupArgs, io: KtxCliIo) => Promise; connection?: (args: KtxConnectionArgs, io: KtxCliIo) => Promise; doctor?: (args: KtxDoctorArgs, io: KtxCliIo) => Promise; diff --git a/packages/cli/src/index.test.ts b/packages/cli/src/index.test.ts index 3c12e583..57dd2104 100644 --- a/packages/cli/src/index.test.ts +++ b/packages/cli/src/index.test.ts @@ -129,9 +129,10 @@ describe('runKtxCli', () => { expect(testIo.stdout()).toContain('Usage: ktx [options] [command]'); expect(testIo.stdout()).toContain('KTX data agent context layer CLI'); - for (const command of ['setup', 'connection', 'ingest', 'wiki', 'sl', 'status', 'dev']) { + for (const command of ['setup', 'connection', 'ingest', 'wiki', 'sl', 'status', 'admin']) { expect(testIo.stdout()).toContain(`${command}`); } + expect(testIo.stdout()).not.toMatch(/^ dev\s/m); expect(testIo.stdout()).not.toMatch(/^ scan\s/m); for (const removed of ['demo', 'init', 'connect', 'ask', 'knowledge', 'agent', 'completion', 'serve']) { expect(testIo.stdout()).not.toMatch(new RegExp(`^\\s+${removed}(?:\\s|\\[|$)`, 'm')); @@ -266,17 +267,17 @@ describe('runKtxCli', () => { const pruneIo = makeIo(); await expect( - runKtxCli(['dev', 'runtime', 'install', '--feature', 'local-embeddings', '--force', '--yes'], installIo.io, { + runKtxCli(['admin', 'runtime', 'install', '--feature', 'local-embeddings', '--force', '--yes'], installIo.io, { runtime, }), ).resolves.toBe(0); await expect( - runKtxCli(['dev', 'runtime', 'start', '--feature', 'local-embeddings', '--force'], startIo.io, { runtime }), + runKtxCli(['admin', 'runtime', 'start', '--feature', 'local-embeddings', '--force'], startIo.io, { runtime }), ).resolves.toBe(0); - await expect(runKtxCli(['dev', 'runtime', 'stop'], stopIo.io, { runtime })).resolves.toBe(0); - await expect(runKtxCli(['dev', 'runtime', 'stop', '--all'], stopAllIo.io, { runtime })).resolves.toBe(0); - await expect(runKtxCli(['dev', 'runtime', 'status', '--json'], statusIo.io, { runtime })).resolves.toBe(0); - await expect(runKtxCli(['dev', 'runtime', 'prune', '--dry-run'], pruneIo.io, { runtime })).resolves.toBe(1); + await expect(runKtxCli(['admin', 'runtime', 'stop'], stopIo.io, { runtime })).resolves.toBe(0); + await expect(runKtxCli(['admin', 'runtime', 'stop', '--all'], stopAllIo.io, { runtime })).resolves.toBe(0); + await expect(runKtxCli(['admin', 'runtime', 'status', '--json'], statusIo.io, { runtime })).resolves.toBe(0); + await expect(runKtxCli(['admin', 'runtime', 'prune', '--dry-run'], pruneIo.io, { runtime })).resolves.toBe(1); expect(runtime).toHaveBeenNthCalledWith( 1, @@ -377,7 +378,7 @@ describe('runKtxCli', () => { it('documents runtime stop all in command help', async () => { const testIo = makeIo(); - await expect(runKtxCli(['dev', 'runtime', 'stop', '--help'], testIo.io)).resolves.toBe(0); + await expect(runKtxCli(['admin', 'runtime', 'stop', '--help'], testIo.io)).resolves.toBe(0); expect(testIo.stdout()).toContain('--all'); expect(testIo.stdout()).toContain('Stop all KTX daemon processes recorded or discoverable'); @@ -655,9 +656,9 @@ describe('runKtxCli', () => { const completionIo = makeIo(); const hiddenIo = makeIo(); - await expect(runKtxCli(['dev', 'completion', 'zsh'], completionIo.io)).resolves.toBe(1); + await expect(runKtxCli(['admin', 'completion', 'zsh'], completionIo.io)).resolves.toBe(1); await expect( - runKtxCli(['dev', '__complete', '--shell', 'zsh', '--position', '2', '--', 'ktx', 'co'], hiddenIo.io), + runKtxCli(['admin', '__complete', '--shell', 'zsh', '--position', '2', '--', 'ktx', 'co'], hiddenIo.io), ).resolves.toBe(1); expect(completionIo.stderr()).toMatch(/unknown command|error:/); @@ -938,7 +939,7 @@ describe('runKtxCli', () => { expect(textIngest).not.toHaveBeenCalled(); }); - it('rejects old adapter-backed ingest flags at the top level and under dev', async () => { + it('rejects old adapter-backed ingest flags at the top level and under admin', async () => { const rootRunIo = makeIo(); const devRunIo = makeIo(); const publicIngest = vi.fn(async () => 0); @@ -949,7 +950,7 @@ describe('runKtxCli', () => { }), ).resolves.toBe(1); await expect( - runKtxCli(['dev', 'ingest', 'run', '--connection-id', 'warehouse', '--adapter', 'metabase'], devRunIo.io, { + runKtxCli(['admin', 'ingest', 'run', '--connection-id', 'warehouse', '--adapter', 'metabase'], devRunIo.io, { publicIngest, }), ).resolves.toBe(1); @@ -958,12 +959,12 @@ describe('runKtxCli', () => { expect(devRunIo.stderr()).toMatch(/unknown command|error:/); }); - it('rejects removed dev doctor and removed ingest parser cases', async () => { + it('rejects removed admin doctor and removed ingest parser cases', async () => { const doctor = vi.fn(async () => 0); const doctorIo = makeIo(); const ingestRunIo = makeIo(); - await expect(runKtxCli(['dev', 'doctor', 'setup', '--json', '--no-input'], doctorIo.io, { doctor })).resolves.toBe(1); + await expect(runKtxCli(['admin', 'doctor', 'setup', '--json', '--no-input'], doctorIo.io, { doctor })).resolves.toBe(1); await expect( runKtxCli( [ @@ -1755,12 +1756,12 @@ describe('runKtxCli', () => { expect(serveIo.stderr()).toMatch(/unknown command|error:/); }); - it('prints dev help for bare dev commands', async () => { + it('prints admin help for bare admin commands', async () => { const testIo = makeIo(); - await expect(runKtxCli(['dev'], testIo.io)).resolves.toBe(0); + await expect(runKtxCli(['admin'], testIo.io)).resolves.toBe(0); - expect(testIo.stdout()).toContain('Usage: ktx dev [options] [command]'); + expect(testIo.stdout()).toContain('Usage: ktx admin [options] [command]'); expect(testIo.stdout()).toContain('Low-level project initialization'); expect(testIo.stdout()).toContain('init'); expect(testIo.stdout()).toContain('runtime'); @@ -1772,13 +1773,13 @@ describe('runKtxCli', () => { expect(testIo.stderr()).toBe(''); }); - it('rejects removed dev command groups without invoking execution', async () => { + it('rejects removed admin command groups without invoking execution', async () => { for (const command of ['scan', 'ingest', 'mapping']) { const testIo = makeIo(); const publicIngest = vi.fn().mockResolvedValue(0); const sl = vi.fn().mockResolvedValue(0); - await expect(runKtxCli(['dev', command], testIo.io, { publicIngest, sl })).resolves.toBe(1); + await expect(runKtxCli(['admin', command], testIo.io, { publicIngest, sl })).resolves.toBe(1); expect(testIo.stderr()).toMatch(/unknown command|error:/); expect(publicIngest).not.toHaveBeenCalled(); @@ -1786,10 +1787,10 @@ describe('runKtxCli', () => { } }); - it('rejects removed reserved dev subcommands', async () => { + it('rejects removed reserved admin subcommands', async () => { const testIo = makeIo(); - await expect(runKtxCli(['dev', 'artifacts'], testIo.io)).resolves.toBe(1); + await expect(runKtxCli(['admin', 'artifacts'], testIo.io)).resolves.toBe(1); expect(testIo.stderr()).toMatch(/unknown command|error:/); }); diff --git a/packages/cli/src/managed-python-command.test.ts b/packages/cli/src/managed-python-command.test.ts index 767d8dd1..717accf4 100644 --- a/packages/cli/src/managed-python-command.test.ts +++ b/packages/cli/src/managed-python-command.test.ts @@ -118,9 +118,9 @@ function makeSpinnerEvents() { describe('managedRuntimeInstallCommand', () => { it('prints the exact command for each managed runtime feature', () => { - expect(managedRuntimeInstallCommand('core')).toBe('ktx dev runtime install --yes'); + expect(managedRuntimeInstallCommand('core')).toBe('ktx admin runtime install --yes'); expect(managedRuntimeInstallCommand('local-embeddings')).toBe( - 'ktx dev runtime install --feature local-embeddings --yes', + 'ktx admin runtime install --feature local-embeddings --yes', ); }); }); @@ -221,7 +221,7 @@ describe('createManagedPythonSemanticLayerComputePort', () => { readStatus: vi.fn(async () => missingStatus()), installRuntime, }), - ).rejects.toThrow('KTX Python runtime is required for this command. Run: ktx dev runtime install --yes'); + ).rejects.toThrow('KTX Python runtime is required for this command. Run: ktx admin runtime install --yes'); expect(installRuntime).not.toHaveBeenCalled(); }); diff --git a/packages/cli/src/managed-python-command.ts b/packages/cli/src/managed-python-command.ts index 11e794ff..83953602 100644 --- a/packages/cli/src/managed-python-command.ts +++ b/packages/cli/src/managed-python-command.ts @@ -53,8 +53,8 @@ export interface ManagedPythonSemanticLayerComputeOptions extends ManagedPythonC export function managedRuntimeInstallCommand(feature: KtxRuntimeFeature): string { return feature === 'local-embeddings' - ? 'ktx dev runtime install --feature local-embeddings --yes' - : 'ktx dev runtime install --yes'; + ? 'ktx admin runtime install --feature local-embeddings --yes' + : 'ktx admin runtime install --yes'; } function installPrompt(feature: KtxRuntimeFeature): string { diff --git a/packages/cli/src/managed-python-runtime.test.ts b/packages/cli/src/managed-python-runtime.test.ts index 13b97a45..92e34e35 100644 --- a/packages/cli/src/managed-python-runtime.test.ts +++ b/packages/cli/src/managed-python-runtime.test.ts @@ -513,7 +513,7 @@ describe('doctorManagedPythonRuntime', () => { ['asset', 'pass'], ['runtime', 'fail'], ]); - expect(checks[2]?.fix).toBe('Run: ktx dev runtime install --yes'); + expect(checks[2]?.fix).toBe('Run: ktx admin runtime install --yes'); }); it('reports uv as a hard prerequisite when uv is missing', async () => { @@ -534,7 +534,7 @@ describe('doctorManagedPythonRuntime', () => { label: 'uv', status: 'fail', detail: MISSING_UV_RUNTIME_INSTALL_MESSAGE, - fix: 'Install uv, make sure it is on PATH, and run: ktx dev runtime install --yes', + fix: 'Install uv, make sure it is on PATH, and run: ktx admin runtime install --yes', }); }); }); diff --git a/packages/cli/src/managed-python-runtime.ts b/packages/cli/src/managed-python-runtime.ts index 88b0fa2b..68272840 100644 --- a/packages/cli/src/managed-python-runtime.ts +++ b/packages/cli/src/managed-python-runtime.ts @@ -122,7 +122,7 @@ export interface ManagedPythonRuntimeDoctorCheck { } export const MISSING_UV_RUNTIME_INSTALL_MESSAGE = - 'uv is required to install the KTX Python runtime. KTX does not download uv automatically. Install uv, make sure it is on PATH, and retry: ktx dev runtime install --yes'; + 'uv is required to install the KTX Python runtime. KTX does not download uv automatically. Install uv, make sure it is on PATH, and retry: ktx admin runtime install --yes'; function defaultAssetDir(): string { return fileURLToPath(new URL('../assets/python/', import.meta.url)); @@ -471,7 +471,7 @@ export async function doctorManagedPythonRuntime( id: 'uv', label: 'uv', detail: error instanceof Error ? error.message : String(error), - fix: 'Install uv, make sure it is on PATH, and run: ktx dev runtime install --yes', + fix: 'Install uv, make sure it is on PATH, and run: ktx admin runtime install --yes', }), ); } @@ -496,7 +496,7 @@ export async function doctorManagedPythonRuntime( id: 'runtime', label: 'Managed Python runtime', detail: status.detail, - ...(status.kind === 'ready' ? {} : { fix: 'Run: ktx dev runtime install --yes' }), + ...(status.kind === 'ready' ? {} : { fix: 'Run: ktx admin runtime install --yes' }), }), ); return checks; diff --git a/packages/cli/src/print-command-tree.test.ts b/packages/cli/src/print-command-tree.test.ts index ececa88c..edd0b69a 100644 --- a/packages/cli/src/print-command-tree.test.ts +++ b/packages/cli/src/print-command-tree.test.ts @@ -12,7 +12,7 @@ describe('renderKtxCommandTree', () => { .filter((line) => /^ {2}[├└]── \S/.test(line)) .map((line) => line.replace(/^ {2}[├└]── /, '').trim().split(' ')[0]); - for (const expected of ['setup', 'connection', 'ingest', 'sl', 'mcp', 'dev']) { + for (const expected of ['setup', 'connection', 'ingest', 'sl', 'mcp', 'admin']) { expect(topLevel).toContain(expected); } diff --git a/packages/cli/src/runtime.test.ts b/packages/cli/src/runtime.test.ts index 01a529e7..e4a7883f 100644 --- a/packages/cli/src/runtime.test.ts +++ b/packages/cli/src/runtime.test.ts @@ -291,7 +291,7 @@ describe('runKtxRuntime', () => { label: 'Managed Python runtime', status: 'fail', detail: 'No runtime manifest at /runtime/0.2.0/manifest.json', - fix: 'Run: ktx dev runtime install --yes', + fix: 'Run: ktx admin runtime install --yes', }, ]), }; diff --git a/packages/cli/src/scan.test.ts b/packages/cli/src/scan.test.ts index b08c15de..8fa51130 100644 --- a/packages/cli/src/scan.test.ts +++ b/packages/cli/src/scan.test.ts @@ -368,8 +368,8 @@ describe('runKtxScan', () => { expect(io.stdout()).toContain('Report: raw-sources/warehouse/live-database/sync-1/scan-report.json'); expect(io.stdout()).toContain('Next:\n'); expect(io.stdout()).toContain('ktx status --project-dir '); - expect(io.stdout()).not.toContain('ktx dev scan status'); - expect(io.stdout()).not.toContain('ktx dev scan report'); + expect(io.stdout()).not.toContain('ktx admin scan status'); + expect(io.stdout()).not.toContain('ktx admin scan report'); expect(io.stdout()).not.toContain('\u001b['); expect(io.stdout()).not.toContain('✓'); expect(io.stdout()).not.toContain('+1'); diff --git a/packages/cli/src/setup-embeddings.test.ts b/packages/cli/src/setup-embeddings.test.ts index 7e22be26..36df256a 100644 --- a/packages/cli/src/setup-embeddings.test.ts +++ b/packages/cli/src/setup-embeddings.test.ts @@ -286,7 +286,7 @@ describe('setup embeddings step', () => { const io = makeIo(); const ensureLocalEmbeddings = vi.fn(async () => { throw new Error( - 'KTX Python runtime is required for this command. Run: ktx dev runtime install --feature local-embeddings --yes', + 'KTX Python runtime is required for this command. Run: ktx admin runtime install --feature local-embeddings --yes', ); }); @@ -304,7 +304,7 @@ describe('setup embeddings step', () => { expect(result.status).toBe('failed'); expect(io.stderr()).toContain( - 'KTX Python runtime is required for this command. Run: ktx dev runtime install --feature local-embeddings --yes', + 'KTX Python runtime is required for this command. Run: ktx admin runtime install --feature local-embeddings --yes', ); }); @@ -331,7 +331,7 @@ describe('setup embeddings step', () => { expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:'); expect(config.ingest.embeddings.backend).toBe('none'); expect(io.stderr()).toContain('Local embedding health check failed: 401 invalid api key [redacted]'); - expect(io.stderr()).toContain('Prepare the runtime with: ktx dev runtime start --feature local-embeddings'); + expect(io.stderr()).toContain('Prepare the runtime with: ktx admin runtime start --feature local-embeddings'); expect(io.stderr()).not.toContain('skip for now'); }); diff --git a/packages/cli/src/setup-embeddings.ts b/packages/cli/src/setup-embeddings.ts index 475e5126..442b67bc 100644 --- a/packages/cli/src/setup-embeddings.ts +++ b/packages/cli/src/setup-embeddings.ts @@ -307,7 +307,7 @@ function localEmbeddingSetupMessage(message: string, stderrTail: string[] = []): const lines = [ `Local embedding health check failed: ${message}`, 'Local embeddings use the KTX-managed Python runtime.', - 'Prepare the runtime with: ktx dev runtime start --feature local-embeddings', + 'Prepare the runtime with: ktx admin runtime start --feature local-embeddings', 'Use --yes with setup to install and start the runtime without prompting.', 'The first run may download Python packages and the all-MiniLM-L6-v2 model.', ]; diff --git a/packages/cli/src/setup-runtime.test.ts b/packages/cli/src/setup-runtime.test.ts index ee070fc7..1d739423 100644 --- a/packages/cli/src/setup-runtime.test.ts +++ b/packages/cli/src/setup-runtime.test.ts @@ -71,7 +71,7 @@ describe('runKtxSetupRuntimeStep', () => { it('fails fast when required runtime features cannot be installed in no-input mode', async () => { const io = makeIo(); const ensureRuntime = vi.fn(async () => { - throw new Error('KTX Python runtime is required for this command. Run: ktx dev runtime install --yes'); + throw new Error('KTX Python runtime is required for this command. Run: ktx admin runtime install --yes'); }); await expect( @@ -94,7 +94,7 @@ describe('runKtxSetupRuntimeStep', () => { expect(ensureRuntime).toHaveBeenCalledWith(expect.objectContaining({ installPolicy: 'never' })); expect((await readKtxSetupState(tempDir)).completed_steps).not.toContain('runtime'); - expect(io.stderr()).toContain('ktx dev runtime install --yes'); + expect(io.stderr()).toContain('ktx admin runtime install --yes'); }); it('starts the managed local embeddings daemon for configured sentence-transformers embeddings', async () => { diff --git a/packages/cli/src/standalone-smoke.test.ts b/packages/cli/src/standalone-smoke.test.ts index 688e69f2..3f580a5e 100644 --- a/packages/cli/src/standalone-smoke.test.ts +++ b/packages/cli/src/standalone-smoke.test.ts @@ -144,6 +144,11 @@ describe('standalone built ktx CLI smoke', () => { expectSetupStderr(init); expect(init.stdout).toContain(`Project: ${projectDir}`); + const reindex = await runBuiltCli(['--project-dir', projectDir, 'admin', 'reindex', '--output', 'plain']); + expect(reindex.code).toBe(0); + expect(reindex.stdout).toContain('reindex\t'); + expect(reindex.stderr).toContain('wiki/global'); + const run = await runBuiltCli([ 'ingest', 'run', diff --git a/packages/context/package.json b/packages/context/package.json index 104b4e47..11cfc28f 100644 --- a/packages/context/package.json +++ b/packages/context/package.json @@ -50,6 +50,11 @@ "import": "./dist/ingest/metabase-mapping.js", "default": "./dist/ingest/metabase-mapping.js" }, + "./index-sync": { + "types": "./dist/index-sync/index.d.ts", + "import": "./dist/index-sync/index.js", + "default": "./dist/index-sync/index.js" + }, "./scan": { "types": "./dist/scan/index.d.ts", "import": "./dist/scan/index.js", diff --git a/packages/context/src/index-sync/index.ts b/packages/context/src/index-sync/index.ts new file mode 100644 index 00000000..5863688e --- /dev/null +++ b/packages/context/src/index-sync/index.ts @@ -0,0 +1,2 @@ +export type { ReindexOptions, ReindexScopeResult, ReindexSummary, ReindexWorkResult } from './types.js'; +export { discoverReindexScopes, reindexLocalIndexes } from './reindex.js'; diff --git a/packages/context/src/index-sync/reindex.test.ts b/packages/context/src/index-sync/reindex.test.ts new file mode 100644 index 00000000..beb62342 --- /dev/null +++ b/packages/context/src/index-sync/reindex.test.ts @@ -0,0 +1,196 @@ +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import type { KtxEmbeddingPort } from '../core/index.js'; +import { initKtxProject, loadKtxProject, type KtxLocalProject } from '../project/index.js'; +import { SqliteKnowledgeIndex } from '../wiki/sqlite-knowledge-index.js'; +import { reindexLocalIndexes } from './reindex.js'; + +class FakeEmbeddingPort implements KtxEmbeddingPort { + readonly maxBatchSize = 8; + + async computeEmbedding(text: string): Promise { + return [text.length, 1]; + } + + async computeEmbeddingsBulk(texts: string[]): Promise { + return texts.map((text) => [text.length, 1]); + } +} + +async function createProject(tempDir: string): Promise { + await initKtxProject({ projectDir: tempDir, force: true }); + return loadKtxProject({ projectDir: tempDir }); +} + +describe('reindexLocalIndexes', () => { + let tempDir: string; + + beforeEach(async () => { + tempDir = await mkdtemp(join(tmpdir(), 'ktx-reindex-')); + }); + + afterEach(async () => { + await rm(tempDir, { recursive: true, force: true }); + }); + + it('returns an empty summary when no wiki or semantic-layer directories exist', async () => { + const project = await createProject(tempDir); + await rm(join(project.projectDir, 'wiki'), { recursive: true, force: true }); + await rm(join(project.projectDir, 'semantic-layer'), { recursive: true, force: true }); + + await expect(reindexLocalIndexes(project, { force: false, embeddingService: null })).resolves.toMatchObject({ + scopes: [], + totals: { scanned: 0, updated: 0, deleted: 0, embeddingsRecomputed: 0, embeddingsFailed: 0 }, + force: false, + embeddingsAvailable: false, + }); + }); + + it('discovers empty directories as zero-row scopes', async () => { + const project = await createProject(tempDir); + await mkdir(join(project.projectDir, 'wiki/user/local'), { recursive: true }); + await mkdir(join(project.projectDir, 'semantic-layer/warehouse'), { recursive: true }); + + const summary = await reindexLocalIndexes(project, { force: false, embeddingService: null }); + + expect(summary.scopes.map((scope) => scope.label)).toEqual(['global', 'user/local', 'warehouse']); + expect(summary.totals.scanned).toBe(0); + }); + + it('indexes mixed wiki and SL sources and reports totals', async () => { + const project = await createProject(tempDir); + await writeFile( + join(project.projectDir, 'wiki/global/revenue.md'), + '---\nsummary: Revenue\nusage_mode: auto\n---\n\nPaid orders.\n', + 'utf-8', + ); + await mkdir(join(project.projectDir, 'semantic-layer/warehouse'), { recursive: true }); + await writeFile( + join(project.projectDir, 'semantic-layer/warehouse/orders.yaml'), + 'name: orders\ntable: public.orders\ngrain: [id]\ncolumns:\n - name: id\n type: number\njoins: []\nmeasures: []\n', + 'utf-8', + ); + + const summary = await reindexLocalIndexes(project, { + force: false, + embeddingService: new FakeEmbeddingPort(), + }); + + expect(summary.scopes).toHaveLength(2); + expect(summary.totals).toMatchObject({ scanned: 2, updated: 2, deleted: 0, embeddingsRecomputed: 2 }); + expect(summary.embeddingsAvailable).toBe(true); + }); + + it('does not report unchanged lexical-only rows as updated on repeated runs', async () => { + const project = await createProject(tempDir); + await writeFile( + join(project.projectDir, 'wiki/global/revenue.md'), + '---\nsummary: Revenue\nusage_mode: auto\n---\n\nPaid orders.\n', + 'utf-8', + ); + await mkdir(join(project.projectDir, 'semantic-layer/warehouse'), { recursive: true }); + await writeFile( + join(project.projectDir, 'semantic-layer/warehouse/orders.yaml'), + 'name: orders\ntable: public.orders\ngrain: [id]\ncolumns:\n - name: id\n type: number\njoins: []\nmeasures: []\n', + 'utf-8', + ); + + const first = await reindexLocalIndexes(project, { force: false, embeddingService: null }); + expect(first.totals).toMatchObject({ + scanned: 2, + updated: 2, + deleted: 0, + embeddingsRecomputed: 0, + embeddingsFailed: 0, + }); + + const second = await reindexLocalIndexes(project, { force: false, embeddingService: null }); + + expect(second.totals).toMatchObject({ + scanned: 2, + updated: 0, + deleted: 0, + embeddingsRecomputed: 0, + embeddingsFailed: 0, + }); + expect(second.scopes.map((scope) => [scope.label, scope.updated])).toEqual([ + ['global', 0], + ['warehouse', 0], + ]); + }); + + it('force clears stale rows before rebuilding each discovered scope', async () => { + const project = await createProject(tempDir); + const wikiIndex = new SqliteKnowledgeIndex({ dbPath: join(project.projectDir, '.ktx/db.sqlite') }); + wikiIndex.sync([ + { + path: 'wiki/global/stale.md', + key: 'stale', + scope: 'GLOBAL', + scopeId: null, + summary: 'Stale', + content: 'Stale content', + tags: [], + embedding: [1, 0], + }, + ]); + await writeFile( + join(project.projectDir, 'wiki/global/revenue.md'), + '---\nsummary: Revenue\nusage_mode: auto\n---\n\nPaid orders.\n', + 'utf-8', + ); + + const summary = await reindexLocalIndexes(project, { + force: true, + embeddingService: new FakeEmbeddingPort(), + }); + + expect(summary.force).toBe(true); + expect(summary.totals).toMatchObject({ scanned: 1, updated: 1, deleted: 0 }); + expect(wikiIndex.search('Stale', 10)).toEqual([]); + }); + + it('captures a per-scope error and continues other scopes', async () => { + const project = await createProject(tempDir); + await writeFile( + join(project.projectDir, 'wiki/global/revenue.md'), + '---\nsummary: Revenue\nusage_mode: auto\n---\n\nPaid orders.\n', + 'utf-8', + ); + await mkdir(join(project.projectDir, 'semantic-layer/warehouse'), { recursive: true }); + await writeFile(join(project.projectDir, 'semantic-layer/warehouse/broken.yaml'), 'not: [valid', 'utf-8'); + + const summary = await reindexLocalIndexes(project, { force: false, embeddingService: null }); + + expect(summary.scopes.find((scope) => scope.label === 'global')?.error).toBeUndefined(); + expect(summary.scopes.find((scope) => scope.label === 'warehouse')?.error).toContain('YAML'); + }); + + it('marks a scope errored when configured embeddings fail', async () => { + const project = await createProject(tempDir); + await writeFile( + join(project.projectDir, 'wiki/global/revenue.md'), + '---\nsummary: Revenue\nusage_mode: auto\n---\n\nPaid orders.\n', + 'utf-8', + ); + const embeddingService: KtxEmbeddingPort = { + maxBatchSize: 8, + async computeEmbedding() { + throw new Error('embedding provider unavailable'); + }, + async computeEmbeddingsBulk() { + throw new Error('embedding provider unavailable'); + }, + }; + + const summary = await reindexLocalIndexes(project, { force: false, embeddingService }); + + expect(summary.scopes[0]).toMatchObject({ + label: 'global', + embeddingsFailed: 1, + error: '1 embedding recomputation failed', + }); + }); +}); diff --git a/packages/context/src/index-sync/reindex.ts b/packages/context/src/index-sync/reindex.ts new file mode 100644 index 00000000..d0cbe29a --- /dev/null +++ b/packages/context/src/index-sync/reindex.ts @@ -0,0 +1,162 @@ +import { readdir, stat } from 'node:fs/promises'; +import { join, relative } from 'node:path'; +import { ktxLocalStateDbPath, type KtxLocalProject } from '../project/index.js'; +import { loadLocalSlSourceRecords, SlSearchService, SqliteSlSourcesIndex } from '../sl/index.js'; +import { KnowledgeWikiService, SqliteKnowledgeIndex } from '../wiki/index.js'; +import type { ReindexOptions, ReindexScopeResult, ReindexSummary, ReindexWorkResult } from './types.js'; + +type DiscoveredScope = + | { kind: 'wiki'; scope: 'GLOBAL'; scopeId: null; label: 'global' } + | { kind: 'wiki'; scope: 'USER'; scopeId: string; label: `user/${string}` } + | { kind: 'sl'; connectionId: string; label: string }; + +const ZERO: ReindexWorkResult = { + scanned: 0, + updated: 0, + deleted: 0, + embeddingsRecomputed: 0, + embeddingsFailed: 0, +}; + +async function directoryExists(path: string): Promise { + try { + return (await stat(path)).isDirectory(); + } catch { + return false; + } +} + +async function childDirectories(path: string): Promise { + try { + const entries = await readdir(path, { withFileTypes: true }); + return entries + .filter((entry) => entry.isDirectory()) + .map((entry) => entry.name) + .sort((left, right) => left.localeCompare(right)); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + return []; + } + throw error; + } +} + +export async function discoverReindexScopes(project: KtxLocalProject): Promise { + const scopes: DiscoveredScope[] = []; + if (await directoryExists(join(project.projectDir, 'wiki/global'))) { + scopes.push({ kind: 'wiki', scope: 'GLOBAL', scopeId: null, label: 'global' }); + } + for (const userId of await childDirectories(join(project.projectDir, 'wiki/user'))) { + scopes.push({ kind: 'wiki', scope: 'USER', scopeId: userId, label: `user/${userId}` }); + } + for (const connectionId of await childDirectories(join(project.projectDir, 'semantic-layer'))) { + if (connectionId !== '_schema') { + scopes.push({ kind: 'sl', connectionId, label: connectionId }); + } + } + return scopes; +} + +function errorMessage(error: unknown): string { + if (!(error instanceof Error)) { + return String(error); + } + return error.name && error.name !== 'Error' ? `${error.name}: ${error.message}` : error.message; +} + +function addTotals(left: ReindexWorkResult, right: ReindexWorkResult): ReindexWorkResult { + return { + scanned: left.scanned + right.scanned, + updated: left.updated + right.updated, + deleted: left.deleted + right.deleted, + embeddingsRecomputed: left.embeddingsRecomputed + right.embeddingsRecomputed, + embeddingsFailed: left.embeddingsFailed + right.embeddingsFailed, + }; +} + +function durationSince(startedAt: bigint): number { + return Number((process.hrtime.bigint() - startedAt) / 1_000_000n); +} + +function embeddingFailureError(work: ReindexWorkResult): string | undefined { + if (work.embeddingsFailed === 0) { + return undefined; + } + return `${work.embeddingsFailed} embedding recomputation${work.embeddingsFailed === 1 ? '' : 's'} failed`; +} + +export async function reindexLocalIndexes( + project: KtxLocalProject, + options: ReindexOptions, +): Promise { + const startedAt = process.hrtime.bigint(); + const dbPath = ktxLocalStateDbPath(project); + const scopes = await discoverReindexScopes(project); + const wikiIndex = new SqliteKnowledgeIndex({ dbPath }); + const slIndex = new SqliteSlSourcesIndex({ dbPath }); + const wikiService = new KnowledgeWikiService(project.fileStore, options.embeddingService, wikiIndex, project.git); + const slService = new SlSearchService(options.embeddingService, slIndex); + const results: ReindexScopeResult[] = []; + + for (const scope of scopes) { + const scopeStartedAt = process.hrtime.bigint(); + try { + let work: ReindexWorkResult; + if (scope.kind === 'wiki') { + if (options.force) { + wikiIndex.clear(scope.scope, scope.scopeId); + } + work = await wikiService.syncIndex(scope.scope, scope.scopeId); + results.push({ + kind: 'wiki', + label: scope.label, + scope: scope.scope === 'GLOBAL' ? 'global' : 'user', + scopeId: scope.scopeId, + ...work, + ...(options.force ? { deleted: 0 } : {}), + ...(options.embeddingService && work.embeddingsFailed > 0 ? { error: embeddingFailureError(work) } : {}), + durationMs: durationSince(scopeStartedAt), + }); + continue; + } + + if (options.force) { + await slIndex.clear(scope.connectionId); + } + const records = await loadLocalSlSourceRecords(project, { connectionId: scope.connectionId }); + work = await slService.indexSources( + scope.connectionId, + records.map((record) => record.source), + ); + results.push({ + kind: 'sl', + label: scope.label, + connectionId: scope.connectionId, + ...work, + ...(options.force ? { deleted: 0 } : {}), + ...(options.embeddingService && work.embeddingsFailed > 0 ? { error: embeddingFailureError(work) } : {}), + durationMs: durationSince(scopeStartedAt), + }); + } catch (error) { + results.push({ + kind: scope.kind, + label: scope.label, + ...(scope.kind === 'wiki' + ? { scope: scope.scope === 'GLOBAL' ? 'global' : 'user', scopeId: scope.scopeId } + : { connectionId: scope.connectionId }), + ...ZERO, + durationMs: durationSince(scopeStartedAt), + error: errorMessage(error), + }); + } + } + + return { + scopes: results, + totals: results.reduce(addTotals, ZERO), + dbPath: relative(project.projectDir, dbPath) || dbPath, + force: options.force, + embeddingsAvailable: options.embeddingService !== null, + durationMs: durationSince(startedAt), + }; +} diff --git a/packages/context/src/index-sync/types.ts b/packages/context/src/index-sync/types.ts new file mode 100644 index 00000000..39970b57 --- /dev/null +++ b/packages/context/src/index-sync/types.ts @@ -0,0 +1,33 @@ +import type { KtxEmbeddingPort } from '../core/index.js'; + +export interface ReindexOptions { + force: boolean; + embeddingService: KtxEmbeddingPort | null; +} + +export interface ReindexWorkResult { + scanned: number; + updated: number; + deleted: number; + embeddingsRecomputed: number; + embeddingsFailed: number; +} + +export interface ReindexScopeResult extends ReindexWorkResult { + kind: 'wiki' | 'sl'; + label: string; + scope?: 'global' | 'user'; + scopeId?: string | null; + connectionId?: string; + durationMs: number; + error?: string; +} + +export interface ReindexSummary { + scopes: ReindexScopeResult[]; + totals: ReindexWorkResult; + dbPath: string; + force: boolean; + embeddingsAvailable: boolean; + durationMs: number; +} diff --git a/packages/context/src/index.ts b/packages/context/src/index.ts index dda42789..1441cad5 100644 --- a/packages/context/src/index.ts +++ b/packages/context/src/index.ts @@ -12,6 +12,7 @@ export * from './agent/index.js'; export * from './core/index.js'; export * from './daemon/index.js'; export * from './ingest/index.js'; +export * from './index-sync/index.js'; export * from './llm/index.js'; export type { CaptureSession, diff --git a/packages/context/src/ingest/local-bundle-runtime.ts b/packages/context/src/ingest/local-bundle-runtime.ts index cb881d37..7e0fc1e5 100644 --- a/packages/context/src/ingest/local-bundle-runtime.ts +++ b/packages/context/src/ingest/local-bundle-runtime.ts @@ -380,16 +380,19 @@ class LocalKnowledgeIndex implements KnowledgeIndexPort { return result; } - async deleteStale(): Promise { + async deleteStale(): Promise { await this.syncAllPagesFromDisk(); + return 0; } - async deleteByScope(): Promise { + async deleteByScope(): Promise { await this.syncAllPagesFromDisk(); + return 0; } - async deleteByKey(): Promise { + async deleteByKey(): Promise { await this.syncAllPagesFromDisk(); + return 0; } async findPageByKey(scope: string, scopeId: string | null, pageKey: string) { diff --git a/packages/context/src/memory/local-memory.ts b/packages/context/src/memory/local-memory.ts index a83c7a08..c12dec74 100644 --- a/packages/context/src/memory/local-memory.ts +++ b/packages/context/src/memory/local-memory.ts @@ -205,11 +205,17 @@ class LocalKnowledgeIndex implements KnowledgeIndexPort { return new Map(); } - async deleteStale(): Promise {} + async deleteStale(): Promise { + return 0; + } - async deleteByScope(): Promise {} + async deleteByScope(): Promise { + return 0; + } - async deleteByKey(): Promise {} + async deleteByKey(): Promise { + return 0; + } async findPageByKey(scope: string, scopeId: string | null, pageKey: string) { const path = this.pagePath(scope, scopeId, pageKey); diff --git a/packages/context/src/sl/ports.ts b/packages/context/src/sl/ports.ts index 888248d5..e01e1e73 100644 --- a/packages/context/src/sl/ports.ts +++ b/packages/context/src/sl/ports.ts @@ -40,9 +40,9 @@ export interface SlSourcesIndexPort { sources: Array<{ sourceName: string; searchText: string; embedding: number[] | null; contentHash?: string | null }>, ): Promise; getExistingSearchTexts(connectionId: string): Promise>; - deleteStale(connectionId: string, keepNames: string[]): Promise; - deleteByConnection(connectionId: string): Promise; - deleteByConnectionAndName(connectionId: string, sourceName: string): Promise; + deleteStale(connectionId: string, keepNames: string[]): Promise; + deleteByConnection(connectionId: string): Promise; + deleteByConnectionAndName(connectionId: string, sourceName: string): Promise; search( connectionId: string, queryEmbedding: number[] | null, diff --git a/packages/context/src/sl/sl-search.service.test.ts b/packages/context/src/sl/sl-search.service.test.ts index ffe27cbc..164c3954 100644 --- a/packages/context/src/sl/sl-search.service.test.ts +++ b/packages/context/src/sl/sl-search.service.test.ts @@ -223,4 +223,73 @@ describe('SlSearchService', () => { }, ]); }); + + it('indexSources reports stats and supports lexical-only indexing', async () => { + const repository = { + upsertSources: vi.fn().mockResolvedValue(undefined), + getExistingSearchTexts: vi.fn().mockResolvedValue( + new Map([ + ['old_source', { searchText: 'old source', hasEmbedding: true }], + ]), + ), + deleteStale: vi.fn().mockResolvedValue(1), + deleteByConnection: vi.fn().mockResolvedValue(0), + deleteByConnectionAndName: vi.fn(), + search: vi.fn(), + }; + const service = new SlSearchService(null, repository); + const source: SemanticLayerSource = { + name: 'orders', + table: 'public.orders', + grain: ['id'], + columns: [{ name: 'id', type: 'number' }], + joins: [], + measures: [], + }; + + await expect(service.indexSources('warehouse', [source])).resolves.toEqual({ + scanned: 1, + updated: 1, + deleted: 1, + embeddingsRecomputed: 0, + embeddingsFailed: 0, + }); + expect(repository.upsertSources).toHaveBeenCalledWith('warehouse', [ + expect.objectContaining({ sourceName: 'orders', embedding: null }), + ]); + }); + + it('does not update unchanged lexical-only SL rows on repeated sync', async () => { + const repository = { + upsertSources: vi.fn().mockResolvedValue(undefined), + getExistingSearchTexts: vi.fn().mockResolvedValue( + new Map([ + ['orders', { searchText: 'orders. table: public.orders. id (number)', hasEmbedding: false }], + ]), + ), + deleteStale: vi.fn().mockResolvedValue(0), + deleteByConnection: vi.fn().mockResolvedValue(0), + deleteByConnectionAndName: vi.fn(), + search: vi.fn(), + }; + const service = new SlSearchService(null, repository); + const source: SemanticLayerSource = { + name: 'orders', + table: 'public.orders', + grain: ['id'], + columns: [{ name: 'id', type: 'number' }], + joins: [], + measures: [], + }; + + await expect(service.indexSources('warehouse', [source])).resolves.toEqual({ + scanned: 1, + updated: 0, + deleted: 0, + embeddingsRecomputed: 0, + embeddingsFailed: 0, + }); + expect(repository.upsertSources).toHaveBeenCalledWith('warehouse', []); + expect(repository.deleteStale).toHaveBeenCalledWith('warehouse', ['orders']); + }); }); diff --git a/packages/context/src/sl/sl-search.service.ts b/packages/context/src/sl/sl-search.service.ts index 68ae1557..0a7ecfb5 100644 --- a/packages/context/src/sl/sl-search.service.ts +++ b/packages/context/src/sl/sl-search.service.ts @@ -1,5 +1,6 @@ import type { KtxEmbeddingPort, KtxLogger } from '../core/index.js'; import { noopLogger } from '../core/index.js'; +import type { ReindexWorkResult } from '../index-sync/types.js'; import { DEFAULT_PRIORITY, resolveDescription } from './descriptions.js'; import { normalizeSemanticLayerDescriptions } from './description-normalization.js'; import type { SlSourcesIndexPort } from './ports.js'; @@ -94,73 +95,71 @@ export function buildSemanticLayerSourceSearchText( export class SlSearchService { constructor( - private readonly embeddingService: KtxEmbeddingPort, + private readonly embeddingService: KtxEmbeddingPort | null, private readonly slSourcesRepository: SlSourcesIndexPort, private readonly logger: KtxLogger = noopLogger, ) {} - async indexSources(connectionId: string, sources: SemanticLayerSource[]): Promise { + async indexSources(connectionId: string, sources: SemanticLayerSource[]): Promise { + const existing = await this.slSourcesRepository.getExistingSearchTexts(connectionId); if (sources.length === 0) { - await this.slSourcesRepository.deleteByConnection(connectionId); - return; + const deleted = await this.slSourcesRepository.deleteByConnection(connectionId); + return { scanned: 0, updated: 0, deleted, embeddingsRecomputed: 0, embeddingsFailed: 0 }; } - // Detect which sources actually changed by comparing search_text - const existing = await this.slSourcesRepository.getExistingSearchTexts(connectionId); const searchTexts = sources.map((s) => this.buildSearchText(s)); + const embeddingService = this.embeddingService; const changedIndices: number[] = []; - for (let i = 0; i < sources.length; i++) { - const prev = existing.get(sources[i].name); - if (!prev || prev.searchText !== searchTexts[i] || !prev.hasEmbedding) { + for (let i = 0; i < sources.length; i += 1) { + const previous = existing.get(sources[i]!.name); + if ( + !previous || + previous.searchText !== searchTexts[i] || + (embeddingService !== null && !previous.hasEmbedding) + ) { changedIndices.push(i); } } - if (changedIndices.length === 0) { - // Still clean up stale sources even if nothing changed - const keepNames = sources.map((s) => s.name); - await this.slSourcesRepository.deleteStale(connectionId, keepNames); - this.logger.log(`SL sources for connection ${connectionId}: all ${sources.length} up to date, 0 reindexed`); - return; - } + let changedEmbeddings: (number[] | null)[] = changedIndices.map(() => null); + let embeddingsRecomputed = 0; + let embeddingsFailed = 0; - // Compute embeddings only for changed sources - const changedTexts = changedIndices.map((i) => searchTexts[i]); - let changedEmbeddings: (number[] | null)[]; - try { - const batchSize = this.embeddingService.maxBatchSize; - const allEmbeddings: number[][] = []; - for (let i = 0; i < changedTexts.length; i += batchSize) { - const batch = changedTexts.slice(i, i + batchSize); - const batchEmbeddings = await this.embeddingService.computeEmbeddingsBulk(batch); - allEmbeddings.push(...batchEmbeddings); + if (embeddingService && changedIndices.length > 0) { + try { + const changedTexts = changedIndices.map((index) => searchTexts[index]!); + const allEmbeddings: number[][] = []; + for (let i = 0; i < changedTexts.length; i += embeddingService.maxBatchSize) { + const batch = changedTexts.slice(i, i + embeddingService.maxBatchSize); + allEmbeddings.push(...(await embeddingService.computeEmbeddingsBulk(batch))); + } + changedEmbeddings = allEmbeddings; + embeddingsRecomputed = allEmbeddings.length; + } catch (error) { + this.logger.warn( + `Failed to compute SL source embeddings: ${error instanceof Error ? error.message : String(error)}`, + ); + embeddingsFailed = changedIndices.length; } - changedEmbeddings = allEmbeddings; - } catch (error) { - this.logger.warn( - `Failed to compute SL source embeddings: ${error instanceof Error ? error.message : String(error)}`, - ); - changedEmbeddings = changedIndices.map(() => null); } - const rows = changedIndices.map((srcIdx, i) => { - return { - sourceName: sources[srcIdx].name, - searchText: searchTexts[srcIdx], - embedding: changedEmbeddings[i], - }; - }); - + const rows = changedIndices.map((sourceIndex, embeddingIndex) => ({ + sourceName: sources[sourceIndex]!.name, + searchText: searchTexts[sourceIndex]!, + embedding: changedEmbeddings[embeddingIndex] ?? null, + })); await this.slSourcesRepository.upsertSources(connectionId, rows); - // Remove sources that no longer exist in YAML - const keepNames = sources.map((s) => s.name); - await this.slSourcesRepository.deleteStale(connectionId, keepNames); - - this.logger.log( - `SL sources for connection ${connectionId}: ${changedIndices.length}/${sources.length} reindexed, ${sources.length - changedIndices.length} unchanged`, - ); + const keepNames = sources.map((source) => source.name); + const deleted = await this.slSourcesRepository.deleteStale(connectionId, keepNames); + return { + scanned: sources.length, + updated: changedIndices.length, + deleted, + embeddingsRecomputed, + embeddingsFailed, + }; } async search( @@ -170,12 +169,14 @@ export class SlSearchService { minRrfScore = 0, ): Promise> { let queryEmbedding: number[] | null = null; - try { - queryEmbedding = await this.embeddingService.computeEmbedding(query); - } catch (error) { - this.logger.warn( - `Failed to compute query embedding, falling back to FTS + trigram: ${error instanceof Error ? error.message : String(error)}`, - ); + if (this.embeddingService) { + try { + queryEmbedding = await this.embeddingService.computeEmbedding(query); + } catch (error) { + this.logger.warn( + `Failed to compute query embedding, falling back to FTS + trigram: ${error instanceof Error ? error.message : String(error)}`, + ); + } } const results = await this.slSourcesRepository.search(connectionId, queryEmbedding, query, limit, minRrfScore); diff --git a/packages/context/src/sl/sqlite-sl-sources-index.test.ts b/packages/context/src/sl/sqlite-sl-sources-index.test.ts index 18258000..91a7727e 100644 --- a/packages/context/src/sl/sqlite-sl-sources-index.test.ts +++ b/packages/context/src/sl/sqlite-sl-sources-index.test.ts @@ -105,6 +105,33 @@ describe('SqliteSlSourcesIndex', () => { expect(await index.search('finance', null, 'revenue', 10)).toEqual([]); }); + it('clear removes sources and dictionary rows for one connection only', async () => { + const index = new SqliteSlSourcesIndex({ dbPath }); + await index.upsertSources('warehouse', [ + { sourceName: 'orders', searchText: 'orders revenue paid', embedding: null }, + ]); + await index.upsertSources('finance', [ + { sourceName: 'invoices', searchText: 'invoices revenue paid', embedding: null }, + ]); + await index.replaceDictionaryEntries('warehouse', [ + { connectionId: 'warehouse', sourceName: 'orders', columnName: 'status', value: 'paid', cardinality: 1 }, + ]); + await index.replaceDictionaryEntries('finance', [ + { connectionId: 'finance', sourceName: 'invoices', columnName: 'status', value: 'paid', cardinality: 1 }, + ]); + + await expect(index.clear('warehouse')).resolves.toBe(1); + + expect(await index.search('warehouse', null, 'revenue', 10)).toEqual([]); + expect(await index.search('finance', null, 'revenue', 10)).toEqual([ + expect.objectContaining({ sourceName: 'invoices' }), + ]); + await expect(index.searchDictionaryCandidates({ connectionIds: ['warehouse'], queryText: 'paid', limit: 10 })) + .resolves.toEqual([]); + await expect(index.searchDictionaryCandidates({ connectionIds: ['finance'], queryText: 'paid', limit: 10 })) + .resolves.toEqual([expect.objectContaining({ connectionId: 'finance', sourceName: 'invoices' })]); + }); + it('returns lane candidates with stable connection-scoped IDs', async () => { const index = new SqliteSlSourcesIndex({ dbPath }); diff --git a/packages/context/src/sl/sqlite-sl-sources-index.ts b/packages/context/src/sl/sqlite-sl-sources-index.ts index f53c8eef..cc078ffe 100644 --- a/packages/context/src/sl/sqlite-sl-sources-index.ts +++ b/packages/context/src/sl/sqlite-sl-sources-index.ts @@ -221,10 +221,9 @@ export class SqliteSlSourcesIndex implements SlSourcesIndexPort { ); } - async deleteStale(connectionId: string, keepNames: string[]): Promise { + async deleteStale(connectionId: string, keepNames: string[]): Promise { if (keepNames.length === 0) { - await this.deleteByConnection(connectionId); - return; + return this.deleteByConnection(connectionId); } const placeholders = keepNames.map(() => '?').join(', '); @@ -257,18 +256,29 @@ export class SqliteSlSourcesIndex implements SlSourcesIndexPort { }); remove(stale.map((row) => row.source_name)); + return stale.length; } - async deleteByConnection(connectionId: string): Promise { + async deleteByConnection(connectionId: string): Promise { + return this.clear(connectionId); + } + + async clear(connectionId: string): Promise { + const rows = this.db + .prepare('SELECT source_name FROM local_sl_sources WHERE connection_id = ?') + .all(connectionId) as Array<{ source_name: string }>; const remove = this.db.transaction(() => { this.db.prepare('DELETE FROM local_sl_sources_fts WHERE connection_id = ?').run(connectionId); this.db.prepare('DELETE FROM local_sl_sources WHERE connection_id = ?').run(connectionId); + this.db.prepare('DELETE FROM local_sl_dictionary_values_fts WHERE connection_id = ?').run(connectionId); + this.db.prepare('DELETE FROM local_sl_dictionary_values WHERE connection_id = ?').run(connectionId); }); remove(); + return rows.length; } - async deleteByConnectionAndName(connectionId: string, sourceName: string): Promise { - this.deleteByConnectionAndNameSync(connectionId, sourceName); + async deleteByConnectionAndName(connectionId: string, sourceName: string): Promise { + return this.deleteByConnectionAndNameSync(connectionId, sourceName); } async replaceDictionaryEntries(connectionId: string, entries: SlDictionaryEntry[]): Promise { @@ -537,7 +547,7 @@ export class SqliteSlSourcesIndex implements SlSourcesIndexPort { .filter((row) => row.rrfScore >= minRrfScore); } - private deleteByConnectionAndNameSync(connectionId: string, sourceName: string): void { + private deleteByConnectionAndNameSync(connectionId: string, sourceName: string): number { const remove = this.db.transaction(() => { this.db .prepare( @@ -548,7 +558,7 @@ export class SqliteSlSourcesIndex implements SlSourcesIndexPort { `, ) .run(connectionId, sourceName); - this.db + const result = this.db .prepare( ` DELETE FROM local_sl_sources @@ -557,7 +567,8 @@ export class SqliteSlSourcesIndex implements SlSourcesIndexPort { `, ) .run(connectionId, sourceName); + return Number(result.changes); }); - remove(); + return remove(); } } diff --git a/packages/context/src/wiki/knowledge-wiki.service.test.ts b/packages/context/src/wiki/knowledge-wiki.service.test.ts index f7bb86e4..d9242f97 100644 --- a/packages/context/src/wiki/knowledge-wiki.service.test.ts +++ b/packages/context/src/wiki/knowledge-wiki.service.test.ts @@ -4,9 +4,9 @@ import { KnowledgeWikiService, type WikiFrontmatter } from './knowledge-wiki.ser function makeService() { const pagesRepository: Record> = { upsertPage: vi.fn().mockResolvedValue(undefined), - deleteByKey: vi.fn().mockResolvedValue(undefined), - deleteByScope: vi.fn().mockResolvedValue(undefined), - deleteStale: vi.fn().mockResolvedValue(undefined), + deleteByKey: vi.fn().mockResolvedValue(0), + deleteByScope: vi.fn().mockResolvedValue(0), + deleteStale: vi.fn().mockResolvedValue(0), getExistingSearchTexts: vi.fn().mockResolvedValue(new Map()), applyDiffTransactional: vi.fn().mockResolvedValue(undefined), }; @@ -50,6 +50,87 @@ function makeService() { const fm: WikiFrontmatter = { summary: 'sum', usage_mode: 'auto' }; +describe('KnowledgeWikiService.syncIndex result stats', () => { + it('reports scanned, updated, deleted, and embedding counts', async () => { + const { service, pagesRepository, embeddingService, configService } = makeService(); + configService.listFiles.mockResolvedValue({ files: ['wiki/global/revenue.md'] }); + configService.readFile.mockResolvedValue({ + content: '---\nsummary: Revenue\nusage_mode: auto\ntags:\n - finance\n---\n\nPaid orders.\n', + }); + pagesRepository.getExistingSearchTexts.mockResolvedValue( + new Map([ + ['old-page', { searchText: 'old', hasEmbedding: true }], + ]), + ); + embeddingService.computeEmbeddingsBulk.mockResolvedValue([[0.1, 0.2, 0.3]]); + pagesRepository.deleteStale.mockResolvedValue(1); + + await expect(service.syncIndex('GLOBAL', null)).resolves.toEqual({ + scanned: 1, + updated: 1, + deleted: 1, + embeddingsRecomputed: 1, + embeddingsFailed: 0, + }); + }); + + it('indexes lexical rows when embeddings are not configured', async () => { + const { pagesRepository, configService, gitService, logger } = makeService(); + const service = new KnowledgeWikiService( + configService as any, + null, + pagesRepository as any, + gitService as any, + logger as any, + ); + configService.listFiles.mockResolvedValue({ files: ['wiki/global/revenue.md'] }); + configService.readFile.mockResolvedValue({ + content: '---\nsummary: Revenue\nusage_mode: auto\n---\n\nPaid orders.\n', + }); + pagesRepository.getExistingSearchTexts.mockResolvedValue(new Map()); + pagesRepository.deleteStale.mockResolvedValue(0); + + const result = await service.syncIndex('GLOBAL', null); + + expect(result.embeddingsRecomputed).toBe(0); + expect(result.embeddingsFailed).toBe(0); + expect(pagesRepository.upsertPage).toHaveBeenCalledWith( + expect.objectContaining({ pageKey: 'revenue', embedding: null }), + ); + }); + + it('does not update unchanged lexical-only wiki rows on repeated sync', async () => { + const { pagesRepository, configService, gitService, logger } = makeService(); + const service = new KnowledgeWikiService( + configService as any, + null, + pagesRepository as any, + gitService as any, + logger as any, + ); + configService.listFiles.mockResolvedValue({ files: ['wiki/global/revenue.md'] }); + configService.readFile.mockResolvedValue({ + content: '---\nsummary: Revenue\nusage_mode: auto\n---\n\nPaid orders.\n', + }); + pagesRepository.getExistingSearchTexts.mockResolvedValue( + new Map([ + ['revenue', { searchText: 'revenue\nRevenue\nPaid orders.', hasEmbedding: false }], + ]), + ); + pagesRepository.deleteStale.mockResolvedValue(0); + + await expect(service.syncIndex('GLOBAL', null)).resolves.toEqual({ + scanned: 1, + updated: 0, + deleted: 0, + embeddingsRecomputed: 0, + embeddingsFailed: 0, + }); + expect(pagesRepository.upsertPage).not.toHaveBeenCalled(); + expect(pagesRepository.deleteStale).toHaveBeenCalledWith('GLOBAL', null, ['revenue']); + }); +}); + describe('KnowledgeWikiService.forWorktree isolation', () => { it('syncSinglePage in worktree scope does not call pagesRepository.upsertPage', async () => { const { service, pagesRepository, embeddingService } = makeService(); diff --git a/packages/context/src/wiki/knowledge-wiki.service.ts b/packages/context/src/wiki/knowledge-wiki.service.ts index c8e276ab..88447c14 100644 --- a/packages/context/src/wiki/knowledge-wiki.service.ts +++ b/packages/context/src/wiki/knowledge-wiki.service.ts @@ -2,6 +2,7 @@ import { createHash } from 'node:crypto'; import YAML from 'yaml'; import type { KtxEmbeddingPort, KtxFileStorePort, KtxLogger } from '../core/index.js'; import { noopLogger } from '../core/index.js'; +import type { ReindexWorkResult } from '../index-sync/types.js'; import { assertFlatWikiKey, isFlatWikiKey } from './keys.js'; import { buildKnowledgeSearchText } from './knowledge-search-text.js'; import type { KnowledgeGitDiffPort, KnowledgeIndexPort, UpsertPageParams } from './ports.js'; @@ -16,7 +17,7 @@ export class KnowledgeWikiService { constructor( private readonly configService: KtxFileStorePort, - private readonly embeddingService: KtxEmbeddingPort, + private readonly embeddingService: KtxEmbeddingPort | null, private readonly pagesRepository: KnowledgeIndexPort, private readonly gitService: KnowledgeGitDiffPort, private readonly logger: KtxLogger = noopLogger, @@ -246,10 +247,12 @@ export class KnowledgeWikiService { const searchText = buildKnowledgeSearchText(pageKey, frontmatter.summary, content, frontmatter.tags); let embedding: number[] | null = null; - try { - embedding = await this.embeddingService.computeEmbedding(searchText); - } catch (err) { - this.logger.warn(`Embedding failed for page "${pageKey}": ${err instanceof Error ? err.message : String(err)}`); + if (this.embeddingService) { + try { + embedding = await this.embeddingService.computeEmbedding(searchText); + } catch (err) { + this.logger.warn(`Embedding failed for page "${pageKey}": ${err instanceof Error ? err.message : String(err)}`); + } } await this.pagesRepository.upsertPage({ @@ -269,14 +272,21 @@ export class KnowledgeWikiService { * Full sync: load all pages from disk for a scope, reindex changed pages, clean stale entries. * Mirrors SlSearchService.indexSources() pattern. */ - async syncIndex(scope: string, scopeId?: string | null): Promise { + async syncIndex(scope: string, scopeId?: string | null): Promise { const pageKeys = await this.listPageKeys(scope, scopeId); + const existing = await this.pagesRepository.getExistingSearchTexts(scope, scopeId ?? null); + if (pageKeys.length === 0) { - await this.pagesRepository.deleteByScope(scope, scopeId ?? null); - return; + const deleted = await this.pagesRepository.deleteByScope(scope, scopeId ?? null); + return { + scanned: 0, + updated: 0, + deleted, + embeddingsRecomputed: 0, + embeddingsFailed: 0, + }; } - // Load and parse all pages const pages: Array<{ pageKey: string; frontmatter: WikiFrontmatter; content: string; searchText: string }> = []; for (const key of pageKeys) { const page = await this.readPage(scope, scopeId, key); @@ -286,58 +296,58 @@ export class KnowledgeWikiService { } } - // Detect changes - const existing = await this.pagesRepository.getExistingSearchTexts(scope, scopeId ?? null); - const changedPages = pages.filter((p) => { - const ex = existing.get(p.pageKey); - return !ex || ex.searchText !== p.searchText || !ex.hasEmbedding; + const embeddingService = this.embeddingService; + const changedPages = pages.filter((page) => { + const previous = existing.get(page.pageKey); + return ( + !previous || + previous.searchText !== page.searchText || + (embeddingService !== null && !previous.hasEmbedding) + ); }); - if (changedPages.length === 0) { - // Still clean up stale - await this.pagesRepository.deleteStale(scope, scopeId ?? null, pageKeys); - this.logger.log(`Wiki sync ${scope}: all ${pages.length} pages up to date`); - return; - } + let embeddings: (number[] | null)[] = changedPages.map(() => null); + let embeddingsRecomputed = 0; + let embeddingsFailed = 0; - // Compute embeddings for changed pages (batched) - const changedTexts = changedPages.map((p) => p.searchText); - let embeddings: (number[] | null)[]; - try { - const batchSize = this.embeddingService.maxBatchSize; - const all: number[][] = []; - for (let i = 0; i < changedTexts.length; i += batchSize) { - const batch = changedTexts.slice(i, i + batchSize); - const batchEmb = await this.embeddingService.computeEmbeddingsBulk(batch); - all.push(...batchEmb); + if (embeddingService && changedPages.length > 0) { + try { + const changedTexts = changedPages.map((page) => page.searchText); + const all: number[][] = []; + for (let i = 0; i < changedTexts.length; i += embeddingService.maxBatchSize) { + const batch = changedTexts.slice(i, i + embeddingService.maxBatchSize); + all.push(...(await embeddingService.computeEmbeddingsBulk(batch))); + } + embeddings = all; + embeddingsRecomputed = all.length; + } catch (err) { + this.logger.warn(`Embedding batch failed during sync: ${err instanceof Error ? err.message : String(err)}`); + embeddingsFailed = changedPages.length; } - embeddings = all; - } catch (err) { - this.logger.warn(`Embedding batch failed during sync: ${err instanceof Error ? err.message : String(err)}`); - embeddings = changedPages.map(() => null); } - // Upsert changed pages - for (let i = 0; i < changedPages.length; i++) { - const p = changedPages[i]; + for (let i = 0; i < changedPages.length; i += 1) { + const page = changedPages[i]!; await this.pagesRepository.upsertPage({ scope, scopeId: scopeId ?? null, - pageKey: p.pageKey, - summary: p.frontmatter.summary, - usageMode: p.frontmatter.usage_mode, - sortOrder: p.frontmatter.sort_order ?? 0, - searchText: p.searchText, - embedding: embeddings[i], + pageKey: page.pageKey, + summary: page.frontmatter.summary, + usageMode: page.frontmatter.usage_mode, + sortOrder: page.frontmatter.sort_order ?? 0, + searchText: page.searchText, + embedding: embeddings[i] ?? null, }); } - // Clean stale entries - await this.pagesRepository.deleteStale(scope, scopeId ?? null, pageKeys); - - this.logger.log( - `Wiki sync ${scope}: ${changedPages.length}/${pages.length} reindexed, ${pages.length - changedPages.length} unchanged`, - ); + const deleted = await this.pagesRepository.deleteStale(scope, scopeId ?? null, pageKeys); + return { + scanned: pages.length, + updated: changedPages.length, + deleted, + embeddingsRecomputed, + embeddingsFailed, + }; } /** @@ -388,12 +398,14 @@ export class KnowledgeWikiService { parsed.frontmatter.tags, ); let embedding: number[] | null = null; - try { - embedding = await this.embeddingService.computeEmbedding(searchText); - } catch (err) { - this.logger.warn( - `[wiki.sync] embedding failed for ${parsedPath.pageKey}: ${err instanceof Error ? err.message : String(err)}`, - ); + if (this.embeddingService) { + try { + embedding = await this.embeddingService.computeEmbedding(searchText); + } catch (err) { + this.logger.warn( + `[wiki.sync] embedding failed for ${parsedPath.pageKey}: ${err instanceof Error ? err.message : String(err)}`, + ); + } } const contentHash = createHash('sha256').update(content).digest('hex'); upserts.push({ diff --git a/packages/context/src/wiki/ports.ts b/packages/context/src/wiki/ports.ts index 075a6c20..6bcb2f17 100644 --- a/packages/context/src/wiki/ports.ts +++ b/packages/context/src/wiki/ports.ts @@ -33,9 +33,9 @@ export interface KnowledgeIndexPort { scope: string, scopeId: string | null, ): Promise>; - deleteStale(scope: string, scopeId: string | null, keepKeys: string[]): Promise; - deleteByScope(scope: string, scopeId: string | null): Promise; - deleteByKey(scope: string, scopeId: string | null, pageKey: string): Promise; + deleteStale(scope: string, scopeId: string | null, keepKeys: string[]): Promise; + deleteByScope(scope: string, scopeId: string | null): Promise; + deleteByKey(scope: string, scopeId: string | null, pageKey: string): Promise; findPageByKey( scope: string, scopeId: string | null, diff --git a/packages/context/src/wiki/sqlite-knowledge-index.test.ts b/packages/context/src/wiki/sqlite-knowledge-index.test.ts index 2a45573d..940e9954 100644 --- a/packages/context/src/wiki/sqlite-knowledge-index.test.ts +++ b/packages/context/src/wiki/sqlite-knowledge-index.test.ts @@ -65,6 +65,35 @@ describe('SqliteKnowledgeIndex', () => { expect(index.search('churn', 10)).toEqual([]); }); + it('clear removes one wiki scope and leaves other scopes intact', async () => { + const index = new SqliteKnowledgeIndex({ dbPath }); + index.sync([ + page({ path: 'wiki/global/revenue.md', key: 'revenue', scope: 'GLOBAL', scopeId: null }), + page({ + path: 'wiki/user/local/revenue.md', + key: 'revenue', + scope: 'USER', + scopeId: 'local', + summary: 'Local revenue', + content: 'Local revenue notes.', + }), + page({ + path: 'wiki/user/alex/revenue.md', + key: 'revenue', + scope: 'USER', + scopeId: 'alex', + summary: 'Alex revenue', + content: 'Alex revenue notes.', + }), + ]); + + expect(index.clear('USER', 'local')).toBe(1); + + expect(index.search('Local', 10)).toEqual([]); + expect(index.search('Alex', 10)).toEqual([expect.objectContaining({ path: 'wiki/user/alex/revenue.md' })]); + expect(index.search('definition', 10)).toEqual([expect.objectContaining({ path: 'wiki/global/revenue.md' })]); + }); + it('exposes existing search text and embedding state for incremental refresh', () => { const index = new SqliteKnowledgeIndex({ dbPath }); index.sync([page({ path: 'wiki/global/revenue.md', key: 'revenue', embedding: [1, 0] })]); diff --git a/packages/context/src/wiki/sqlite-knowledge-index.ts b/packages/context/src/wiki/sqlite-knowledge-index.ts index 7a5ae8fc..66b30338 100644 --- a/packages/context/src/wiki/sqlite-knowledge-index.ts +++ b/packages/context/src/wiki/sqlite-knowledge-index.ts @@ -3,6 +3,7 @@ import { dirname } from 'node:path'; import Database from 'better-sqlite3'; import { buildKnowledgeSearchText } from './knowledge-search-text.js'; import type { LocalKnowledgeScope } from './local-knowledge.js'; +import type { KnowledgeIndexPageListing, UpsertPageParams } from './ports.js'; export interface SqliteKnowledgeIndexOptions { dbPath: string; @@ -12,6 +13,7 @@ export interface SqliteKnowledgeIndexPage { path: string; key: string; scope: LocalKnowledgeScope; + scopeId?: string | null; summary: string; content: string; tags: string[]; @@ -106,6 +108,7 @@ export class SqliteKnowledgeIndex { path TEXT PRIMARY KEY, key TEXT NOT NULL, scope TEXT NOT NULL, + scope_id TEXT, summary TEXT NOT NULL, content TEXT NOT NULL, tags TEXT NOT NULL, @@ -129,6 +132,9 @@ export class SqliteKnowledgeIndex { if (!columnNames.has('embedding_json')) { this.db.exec('ALTER TABLE knowledge_pages ADD COLUMN embedding_json TEXT'); } + if (!columnNames.has('scope_id')) { + this.db.exec('ALTER TABLE knowledge_pages ADD COLUMN scope_id TEXT'); + } } sync(pages: SqliteKnowledgeIndexPage[]): void { @@ -142,11 +148,12 @@ export class SqliteKnowledgeIndex { ? this.db.prepare('DELETE FROM knowledge_pages_fts') : this.db.prepare(`DELETE FROM knowledge_pages_fts WHERE path NOT IN (${keepPaths.map(() => '?').join(', ')})`); const upsertPage = this.db.prepare(` - INSERT INTO knowledge_pages (path, key, scope, summary, content, tags, search_text, embedding_json) - VALUES (@path, @key, @scope, @summary, @content, @tags, @searchText, @embeddingJson) + INSERT INTO knowledge_pages (path, key, scope, scope_id, summary, content, tags, search_text, embedding_json) + VALUES (@path, @key, @scope, @scopeId, @summary, @content, @tags, @searchText, @embeddingJson) ON CONFLICT(path) DO UPDATE SET key = excluded.key, scope = excluded.scope, + scope_id = excluded.scope_id, summary = excluded.summary, content = excluded.content, tags = excluded.tags, @@ -168,6 +175,7 @@ export class SqliteKnowledgeIndex { path: page.path, key: page.key, scope: page.scope, + scopeId: page.scopeId ?? null, summary: page.summary, content: searchText, tags: page.tags.join(' '), @@ -275,4 +283,201 @@ export class SqliteKnowledgeIndex { score: scoreFromRank(row.rawScore), })); } + + private pathForPage(scope: string, scopeId: string | null, pageKey: string): string { + return scope === 'GLOBAL' ? `wiki/global/${pageKey}.md` : `wiki/user/${scopeId ?? 'local'}/${pageKey}.md`; + } + + async upsertPage(params: UpsertPageParams): Promise { + const path = this.pathForPage(params.scope, params.scopeId, params.pageKey); + const row = { + path, + key: params.pageKey, + scope: params.scope, + scopeId: params.scopeId, + summary: params.summary, + content: params.searchText, + tags: '', + searchText: params.searchText, + embeddingJson: params.embedding && params.embedding.length > 0 ? JSON.stringify(params.embedding) : null, + }; + const write = this.db.transaction(() => { + this.db + .prepare( + ` + INSERT INTO knowledge_pages (path, key, scope, scope_id, summary, content, tags, search_text, embedding_json) + VALUES (@path, @key, @scope, @scopeId, @summary, @content, @tags, @searchText, @embeddingJson) + ON CONFLICT(path) DO UPDATE SET + key = excluded.key, + scope = excluded.scope, + scope_id = excluded.scope_id, + summary = excluded.summary, + content = excluded.content, + tags = excluded.tags, + search_text = excluded.search_text, + embedding_json = excluded.embedding_json + `, + ) + .run(row); + this.db.prepare('DELETE FROM knowledge_pages_fts WHERE path = @path').run(row); + this.db + .prepare( + ` + INSERT INTO knowledge_pages_fts (path, key, summary, content, tags) + VALUES (@path, @key, @summary, @content, @tags) + `, + ) + .run(row); + }); + write(); + } + + async getExistingSearchTexts( + scope: string, + scopeId: string | null, + ): Promise> { + const rows = this.db + .prepare( + ` + SELECT key, search_text, embedding_json + FROM knowledge_pages + WHERE scope = ? + AND scope_id IS ? + ORDER BY key ASC + `, + ) + .all(scope, scopeId) as Array<{ key: string; search_text: string; embedding_json: string | null }>; + return new Map( + rows.map((row) => [row.key, { searchText: row.search_text, hasEmbedding: row.embedding_json !== null }]), + ); + } + + async deleteStale(scope: string, scopeId: string | null, keepKeys: string[]): Promise { + if (keepKeys.length === 0) { + return this.deleteByScope(scope, scopeId); + } + const placeholders = keepKeys.map(() => '?').join(', '); + const stale = this.db + .prepare( + ` + SELECT key + FROM knowledge_pages + WHERE scope = ? + AND scope_id IS ? + AND key NOT IN (${placeholders}) + `, + ) + .all(scope, scopeId, ...keepKeys) as Array<{ key: string }>; + for (const row of stale) { + await this.deleteByKey(scope, scopeId, row.key); + } + return stale.length; + } + + async deleteByScope(scope: string, scopeId: string | null): Promise { + return this.clear(scope, scopeId); + } + + async deleteByKey(scope: string, scopeId: string | null, pageKey: string): Promise { + const path = this.pathForPage(scope, scopeId, pageKey); + const remove = this.db.transaction(() => { + this.db.prepare('DELETE FROM knowledge_pages_fts WHERE path = ?').run(path); + const result = this.db.prepare('DELETE FROM knowledge_pages WHERE path = ?').run(path); + return Number(result.changes); + }); + return remove(); + } + + clear(scope: string, scopeId: string | null): number { + const rows = this.db + .prepare('SELECT path FROM knowledge_pages WHERE scope = ? AND scope_id IS ?') + .all(scope, scopeId) as Array<{ path: string }>; + const remove = this.db.transaction((paths: string[]) => { + for (const path of paths) { + this.db.prepare('DELETE FROM knowledge_pages_fts WHERE path = ?').run(path); + this.db.prepare('DELETE FROM knowledge_pages WHERE path = ?').run(path); + } + }); + remove(rows.map((row) => row.path)); + return rows.length; + } + + async applyDiffTransactional(params: { + runId: string; + upserts: UpsertPageParams[]; + deletes: Array<{ scope: string; scopeId: string | null; pageKey: string }>; + }): Promise { + void params.runId; + for (const page of params.upserts) { + await this.upsertPage(page); + } + for (const page of params.deletes) { + await this.deleteByKey(page.scope, page.scopeId, page.pageKey); + } + } + + async findPageByKey( + scope: string, + scopeId: string | null, + pageKey: string, + ): Promise<{ id?: string; page_key: string } | null> { + const path = this.pathForPage(scope, scopeId, pageKey); + const row = this.db.prepare('SELECT path, key FROM knowledge_pages WHERE path = ?').get(path) as + | { path: string; key: string } + | undefined; + return row ? { id: row.path, page_key: row.key } : null; + } + + async listPagesForUser(userId: string): Promise { + const rows = this.db + .prepare( + ` + SELECT path, key, scope, scope_id, summary, tags + FROM knowledge_pages + WHERE scope = 'GLOBAL' + OR (scope = 'USER' AND scope_id = ?) + ORDER BY scope ASC, key ASC + `, + ) + .all(userId) as Array<{ + path: string; + key: string; + scope: string; + scope_id: string | null; + summary: string; + tags: string; + }>; + return rows.map((row) => ({ + id: row.path, + page_key: row.key, + summary: row.summary, + scope: row.scope, + scope_id: row.scope_id, + tags: row.tags.split(/\s+/).filter(Boolean), + })); + } + + async getUserPageCount(userId: string): Promise { + const row = this.db + .prepare("SELECT COUNT(*) AS count FROM knowledge_pages WHERE scope = 'USER' AND scope_id = ?") + .get(userId) as { count: number }; + return row.count; + } + + async incrementUsageCount(): Promise {} + + async searchRRF( + userId: string, + _embedding: number[] | null, + queryText: string, + limit: number, + ): Promise> { + const allowedPages = new Map((await this.listPagesForUser(userId)).map((page) => [page.id, page])); + return this.search(queryText, limit) + .map((row) => { + const page = allowedPages.get(row.path); + return page ? { pageKey: page.page_key, summary: page.summary, rrfScore: row.score } : null; + }) + .filter((row): row is { pageKey: string; summary: string; rrfScore: number } => row !== null); + } } diff --git a/scripts/examples-docs.test.mjs b/scripts/examples-docs.test.mjs index bebe4965..1bd35cc3 100644 --- a/scripts/examples-docs.test.mjs +++ b/scripts/examples-docs.test.mjs @@ -193,12 +193,12 @@ describe('standalone example docs', () => { assert.match(rootReadme, publicPackagePattern('npm install -g {package}')); assert.match(quickstart, publicPackagePattern('npm install -g {package}')); - assert.match(quickstart, /ktx dev runtime install --feature local-embeddings --yes/); - assert.match(quickstart, /ktx dev runtime start --feature local-embeddings/); + assert.match(quickstart, /ktx admin runtime install --feature local-embeddings --yes/); + assert.match(quickstart, /ktx admin runtime start --feature local-embeddings/); assert.match(packageArtifacts, /requires `uv` on `PATH`/); - assert.match(packageArtifacts, /ktx dev runtime status/); - assert.match(packageArtifacts, /ktx dev runtime status/); - assert.doesNotMatch(packageArtifacts, /ktx dev runtime prune/); + assert.match(packageArtifacts, /ktx admin runtime status/); + assert.match(packageArtifacts, /ktx admin runtime status/); + assert.doesNotMatch(packageArtifacts, /ktx admin runtime prune/); assert.match( packageArtifacts, new RegExp( @@ -229,9 +229,9 @@ describe('standalone example docs', () => { assert.doesNotMatch(readme, /standalone Python distributions/); assert.doesNotMatch(readme, /installs the Python artifacts directly/); assert.match(readme, /requires `uv` on `PATH`/); - assert.match(readme, /ktx dev runtime status/); - assert.match(readme, /ktx dev runtime status/); - assert.doesNotMatch(readme, /ktx dev runtime prune/); + assert.match(readme, /ktx admin runtime status/); + assert.match(readme, /ktx admin runtime status/); + assert.doesNotMatch(readme, /ktx admin runtime prune/); assert.doesNotMatch(readme, /@ktx\/context/); assert.doesNotMatch(readme, /@ktx\/cli/); assert.doesNotMatch(readme, /python -m ktx_daemon semantic-validate/); @@ -241,7 +241,7 @@ describe('standalone example docs', () => { const rootReadme = await readText('README.md'); const cliMeta = await readText('docs-site/content/docs/cli-reference/meta.json'); const ingestReference = await readText('docs-site/content/docs/cli-reference/ktx-ingest.mdx'); - const devReference = await readText('docs-site/content/docs/cli-reference/ktx-dev.mdx'); + const adminReference = await readText('docs-site/content/docs/cli-reference/ktx-admin.mdx'); const setupReference = await readText('docs-site/content/docs/cli-reference/ktx-setup.mdx'); const buildingContext = await readText('docs-site/content/docs/guides/building-context.mdx'); const contextSources = await readText('docs-site/content/docs/integrations/context-sources.mdx'); @@ -275,7 +275,7 @@ describe('standalone example docs', () => { assert.doesNotMatch(ingestReference, /--adapter/); assert.doesNotMatch(ingestReference, /ktx ingest watch/); assert.doesNotMatch(ingestReference, /live-database/); - assert.doesNotMatch(devReference, /ktx scan/); + assert.doesNotMatch(adminReference, /ktx scan/); assert.doesNotMatch(buildingContext, /ktx ingest watch/); assert.doesNotMatch(buildingContext, /ktx ingest status/); assert.doesNotMatch(buildingContext, /ktx ingest replay/); diff --git a/scripts/installed-live-database-smoke.mjs b/scripts/installed-live-database-smoke.mjs index 6337a582..3675016e 100644 --- a/scripts/installed-live-database-smoke.mjs +++ b/scripts/installed-live-database-smoke.mjs @@ -237,17 +237,17 @@ function parseDaemonBaseUrl(stdout) { } async function startDaemon(cleanInstallDir) { - const result = await run('pnpm', ['exec', 'ktx', 'dev', 'runtime', 'start'], { + const result = await run('pnpm', ['exec', 'ktx', 'admin', 'runtime', 'start'], { cwd: cleanInstallDir, env: managedRuntimeEnv(cleanInstallDir), timeout: 120_000, }); - requireSuccess('ktx dev runtime start', result); + requireSuccess('ktx admin runtime start', result); return parseDaemonBaseUrl(result.stdout); } async function stopDaemon(cleanInstallDir) { - await run('pnpm', ['exec', 'ktx', 'dev', 'runtime', 'stop'], { + await run('pnpm', ['exec', 'ktx', 'admin', 'runtime', 'stop'], { cwd: cleanInstallDir, env: managedRuntimeEnv(cleanInstallDir), timeout: 30_000, @@ -271,7 +271,7 @@ async function prepareCleanInstall(layout, cleanInstallDir) { await run('pnpm', ['install'], { cwd: cleanInstallDir, timeout: 120_000 }).then((result) => requireSuccess('pnpm install clean artifact project', result), ); - await run('pnpm', ['exec', 'ktx', 'dev', 'runtime', 'install', '--yes'], { + await run('pnpm', ['exec', 'ktx', 'admin', 'runtime', 'install', '--yes'], { cwd: cleanInstallDir, env: managedRuntimeEnv(cleanInstallDir), timeout: 120_000, diff --git a/scripts/local-embeddings-runtime-smoke.mjs b/scripts/local-embeddings-runtime-smoke.mjs index 652a039c..6c9c6ed0 100644 --- a/scripts/local-embeddings-runtime-smoke.mjs +++ b/scripts/local-embeddings-runtime-smoke.mjs @@ -74,27 +74,27 @@ export function localEmbeddingsSmokeCommands(input) { timeoutMs: 60_000, }, { - label: 'ktx dev runtime status missing', + label: 'ktx admin runtime status missing', command: 'pnpm', - args: ['exec', 'ktx', 'dev', 'runtime', 'status', '--json'], + args: ['exec', 'ktx', 'admin', 'runtime', 'status', '--json'], timeoutMs: 60_000, }, { - label: 'ktx dev runtime install local embeddings', + label: 'ktx admin runtime install local embeddings', command: 'pnpm', - args: ['exec', 'ktx', 'dev', 'runtime', 'install', '--feature', 'local-embeddings', '--yes'], + args: ['exec', 'ktx', 'admin', 'runtime', 'install', '--feature', 'local-embeddings', '--yes'], timeoutMs: 1_200_000, }, { - label: 'ktx dev runtime status local embeddings ready', + label: 'ktx admin runtime status local embeddings ready', command: 'pnpm', - args: ['exec', 'ktx', 'dev', 'runtime', 'status', '--json'], + args: ['exec', 'ktx', 'admin', 'runtime', 'status', '--json'], timeoutMs: 60_000, }, { - label: 'ktx dev runtime start local embeddings', + label: 'ktx admin runtime start local embeddings', command: 'pnpm', - args: ['exec', 'ktx', 'dev', 'runtime', 'start', '--feature', 'local-embeddings'], + args: ['exec', 'ktx', 'admin', 'runtime', 'start', '--feature', 'local-embeddings'], timeoutMs: 300_000, }, { @@ -118,9 +118,9 @@ export function localEmbeddingsSmokeCommands(input) { timeoutMs: 900_000, }, { - label: 'ktx dev runtime stop local embeddings', + label: 'ktx admin runtime stop local embeddings', command: 'pnpm', - args: ['exec', 'ktx', 'dev', 'runtime', 'stop'], + args: ['exec', 'ktx', 'admin', 'runtime', 'stop'], timeoutMs: 60_000, }, ]; @@ -374,7 +374,7 @@ export async function runLocalEmbeddingsRuntimeSmoke(options = {}) { process.stdout.write('KTX local embeddings runtime smoke verified\n'); } finally { if (daemonStarted) { - await run('pnpm', ['exec', 'ktx', 'dev', 'runtime', 'stop'], { + await run('pnpm', ['exec', 'ktx', 'admin', 'runtime', 'stop'], { cwd: installDir, env: smokeEnv, timeoutMs: 60_000, diff --git a/scripts/local-embeddings-runtime-smoke.test.mjs b/scripts/local-embeddings-runtime-smoke.test.mjs index f059f7a2..1b5cbfdd 100644 --- a/scripts/local-embeddings-runtime-smoke.test.mjs +++ b/scripts/local-embeddings-runtime-smoke.test.mjs @@ -89,23 +89,23 @@ describe('localEmbeddingsSmokeCommands', () => { assert.deepEqual(commands.map((command) => command.label), [ 'ktx public package version', - 'ktx dev runtime status missing', - 'ktx dev runtime install local embeddings', - 'ktx dev runtime status local embeddings ready', - 'ktx dev runtime start local embeddings', + 'ktx admin runtime status missing', + 'ktx admin runtime install local embeddings', + 'ktx admin runtime status local embeddings ready', + 'ktx admin runtime start local embeddings', 'ktx setup local embeddings', - 'ktx dev runtime stop local embeddings', + 'ktx admin runtime stop local embeddings', ]); assert.deepEqual(commands[2], { - label: 'ktx dev runtime install local embeddings', + label: 'ktx admin runtime install local embeddings', command: 'pnpm', - args: ['exec', 'ktx', 'dev', 'runtime', 'install', '--feature', 'local-embeddings', '--yes'], + args: ['exec', 'ktx', 'admin', 'runtime', 'install', '--feature', 'local-embeddings', '--yes'], timeoutMs: 1_200_000, }); assert.deepEqual(commands[4], { - label: 'ktx dev runtime start local embeddings', + label: 'ktx admin runtime start local embeddings', command: 'pnpm', - args: ['exec', 'ktx', 'dev', 'runtime', 'start', '--feature', 'local-embeddings'], + args: ['exec', 'ktx', 'admin', 'runtime', 'start', '--feature', 'local-embeddings'], timeoutMs: 300_000, }); assert.deepEqual(commands[5].args, [ diff --git a/scripts/package-artifacts.mjs b/scripts/package-artifacts.mjs index 3c885ac9..34a5c9a7 100644 --- a/scripts/package-artifacts.mjs +++ b/scripts/package-artifacts.mjs @@ -147,7 +147,7 @@ export async function findPythonArtifacts(pythonDir) { files, RUNTIME_WHEEL_DISTRIBUTION_NAME, '.whl', - 'kaelio-ktx dev runtime wheel', + 'kaelio-ktx runtime wheel', pythonDir, RUNTIME_WHEEL_PACKAGE_VERSION, ), @@ -606,8 +606,8 @@ try { requireOutput('ktx public package version', version, await installedPackageVersionPattern()); const runtimeStatusBefore = parseJsonResultWithExitCode( - 'ktx dev runtime status missing', - await run('pnpm', ['exec', 'ktx', 'dev', 'runtime', 'status', '--json']), + 'ktx admin runtime status missing', + await run('pnpm', ['exec', 'ktx', 'admin', 'runtime', 'status', '--json']), 1, ); assert.equal(runtimeStatusBefore.kind, 'missing'); @@ -768,8 +768,8 @@ try { requireOutput('ktx sl query first managed runtime install', slQuery, /orders/); const runtimeStatusAfter = parseJsonResult( - 'ktx dev runtime status ready', - await run('pnpm', ['exec', 'ktx', 'dev', 'runtime', 'status', '--json']), + 'ktx admin runtime status ready', + await run('pnpm', ['exec', 'ktx', 'admin', 'runtime', 'status', '--json']), ); assert.equal(runtimeStatusAfter.kind, 'ready'); assert.deepEqual(runtimeStatusAfter.manifest.features, ['core']); @@ -797,29 +797,29 @@ try { requireOutput('ktx sl query sqlite execute', sqliteSlQuery, /"rows": \\[\\s*\\[\\s*3\\s*\\]\\s*\\]/); process.stdout.write('ktx sl query sqlite execute verified\\n'); - const runtimeDoctor = await run('pnpm', ['exec', 'ktx', 'dev', 'runtime', 'status']); - requireSuccess('ktx dev runtime status', runtimeDoctor); - requireOutput('ktx dev runtime status', runtimeDoctor, /KTX Python runtime/); - requireOutput('ktx dev runtime status', runtimeDoctor, /status: ready/); - process.stdout.write('ktx dev runtime status verified\\n'); + const runtimeDoctor = await run('pnpm', ['exec', 'ktx', 'admin', 'runtime', 'status']); + requireSuccess('ktx admin runtime status', runtimeDoctor); + requireOutput('ktx admin runtime status', runtimeDoctor, /KTX Python runtime/); + requireOutput('ktx admin runtime status', runtimeDoctor, /status: ready/); + process.stdout.write('ktx admin runtime status verified\\n'); - const runtimeStart = await run('pnpm', ['exec', 'ktx', 'dev', 'runtime', 'start']); - requireSuccess('ktx dev runtime start', runtimeStart); + const runtimeStart = await run('pnpm', ['exec', 'ktx', 'admin', 'runtime', 'start']); + requireSuccess('ktx admin runtime start', runtimeStart); daemonStarted = true; - requireOutput('ktx dev runtime start', runtimeStart, /Started KTX Python daemon/); - requireOutput('ktx dev runtime start', runtimeStart, /url: http:\\/\\/127\\.0\\.0\\.1:\\d+/); - requireOutput('ktx dev runtime start', runtimeStart, /features: core/); + requireOutput('ktx admin runtime start', runtimeStart, /Started KTX Python daemon/); + requireOutput('ktx admin runtime start', runtimeStart, /url: http:\\/\\/127\\.0\\.0\\.1:\\d+/); + requireOutput('ktx admin runtime start', runtimeStart, /features: core/); - const runtimeStartReuse = await run('pnpm', ['exec', 'ktx', 'dev', 'runtime', 'start']); - requireSuccess('ktx dev runtime start reuse', runtimeStartReuse); - requireOutput('ktx dev runtime start reuse', runtimeStartReuse, /Using existing KTX Python daemon/); - requireOutput('ktx dev runtime start reuse', runtimeStartReuse, /features: core/); + const runtimeStartReuse = await run('pnpm', ['exec', 'ktx', 'admin', 'runtime', 'start']); + requireSuccess('ktx admin runtime start reuse', runtimeStartReuse); + requireOutput('ktx admin runtime start reuse', runtimeStartReuse, /Using existing KTX Python daemon/); + requireOutput('ktx admin runtime start reuse', runtimeStartReuse, /features: core/); - const runtimeStop = await run('pnpm', ['exec', 'ktx', 'dev', 'runtime', 'stop']); - requireSuccess('ktx dev runtime stop', runtimeStop); + const runtimeStop = await run('pnpm', ['exec', 'ktx', 'admin', 'runtime', 'stop']); + requireSuccess('ktx admin runtime stop', runtimeStop); daemonStarted = false; - requireOutput('ktx dev runtime stop', runtimeStop, /Stopped KTX Python daemon/); - process.stdout.write('ktx dev runtime daemon lifecycle verified\\n'); + requireOutput('ktx admin runtime stop', runtimeStop, /Stopped KTX Python daemon/); + process.stdout.write('ktx admin runtime daemon lifecycle verified\\n'); const structuralScan = await run('pnpm', ['exec', 'ktx', 'ingest', 'warehouse', '--project-dir', @@ -849,7 +849,7 @@ try { process.stdout.write('ktx ingest state verified\\n'); } finally { if (daemonStarted) { - await run('pnpm', ['exec', 'ktx', 'dev', 'runtime', 'stop']); + await run('pnpm', ['exec', 'ktx', 'admin', 'runtime', 'stop']); } if (previousRuntimeRoot === undefined) { delete process.env.KTX_RUNTIME_ROOT; diff --git a/scripts/package-artifacts.test.mjs b/scripts/package-artifacts.test.mjs index 89d1a760..3c6c4d7e 100644 --- a/scripts/package-artifacts.test.mjs +++ b/scripts/package-artifacts.test.mjs @@ -167,7 +167,7 @@ describe('findPythonArtifacts', () => { it('throws when a required Python artifact is missing', async () => { const root = await mkdtemp(join(tmpdir(), 'ktx-artifacts-test-')); try { - await assert.rejects(() => findPythonArtifacts(root), /Missing Python artifact: kaelio-ktx dev runtime wheel/); + await assert.rejects(() => findPythonArtifacts(root), /Missing Python artifact: kaelio-ktx runtime wheel/); } finally { await rm(root, { recursive: true, force: true }); } @@ -491,20 +491,20 @@ describe('verification snippets', () => { assert.doesNotMatch(source, /run\('python'/); assert.match(source, /KTX_RUNTIME_ROOT/); assert.match(source, /managed-runtime/); - assert.match(source, /ktx dev runtime status missing/); + assert.match(source, /ktx admin runtime status missing/); assert.match(source, /runtimeStatusBefore\.kind, 'missing'/); assert.ok(source.includes(String.raw`Installing KTX Python runtime \(core\) with uv`)); assert.match(source, /KTX Python runtime ready:/); - assert.match(source, /ktx dev runtime status ready/); + assert.match(source, /ktx admin runtime status ready/); assert.match(source, /runtimeStatusAfter\.kind, 'ready'/); assert.match(source, /runtimeStatusAfter\.manifest\.features/); - assert.match(source, /ktx dev runtime status/); + assert.match(source, /ktx admin runtime status/); assert.match(source, /status: ready/); - assert.match(source, /ktx dev runtime start/); - assert.match(source, /ktx dev runtime start reuse/); + assert.match(source, /ktx admin runtime start/); + assert.match(source, /ktx admin runtime start reuse/); assert.match(source, /Using existing KTX Python daemon/); - assert.match(source, /ktx dev runtime stop/); - assert.doesNotMatch(source, /ktx dev runtime prune/); + assert.match(source, /ktx admin runtime stop/); + assert.doesNotMatch(source, /ktx admin runtime prune/); assert.doesNotMatch(source, /staleRuntimeDir/); assert.match(source, /run\('pnpm', \[\s*'exec',\s*'ktx',\s*'ingest',\s*'warehouse'/); assert.match(source, /'--deep'/);