diff --git a/docs-site/content/docs/cli-reference/ktx-sl.mdx b/docs-site/content/docs/cli-reference/ktx-sl.mdx index f5a31b27..b3e5305f 100644 --- a/docs-site/content/docs/cli-reference/ktx-sl.mdx +++ b/docs-site/content/docs/cli-reference/ktx-sl.mdx @@ -1,6 +1,6 @@ --- title: "ktx sl" -description: "List, read, validate, query, or write semantic-layer sources." +description: "List, search, validate, or query semantic-layer sources." --- Interact with your project's semantic layer. Semantic sources are YAML definitions that describe your tables, columns, measures, joins, and grain — the vocabulary agents use to generate correct SQL. @@ -16,9 +16,8 @@ ktx sl [options] | Subcommand | Description | |-----------|-------------| | `list` | List semantic-layer sources | -| `read ` | Read a semantic-layer source | +| `search ` | Search semantic-layer sources | | `validate ` | Validate a semantic-layer source against the database schema | -| `write ` | Write a semantic-layer source | | `query` | Compile or execute a semantic-layer query | ## Options @@ -28,16 +27,17 @@ ktx sl [options] | Flag | Description | Default | |------|-------------|---------| | `--connection-id ` | Filter by KTX connection id | — | -| `--query ` | Search source names and descriptions | — | | `--output ` | Output mode: `pretty` (default in TTY), `plain` (TSV), or `json` | `pretty` | | `--json` | Shortcut for `--output=json` (overrides `--output`) | `false` | -### `sl read` +### `sl search` | Flag | Description | Default | |------|-------------|---------| -| `--connection-id ` | KTX connection id (required) | — | -| `--json` | Print JSON output | `false` | +| `--connection-id ` | Filter by KTX connection id | — | +| `--limit ` | Maximum search results | — | +| `--output ` | Output mode: `pretty` (default in TTY), `plain` (TSV), or `json` | `pretty` | +| `--json` | Shortcut for `--output=json` (overrides `--output`) | `false` | ### `sl validate` @@ -45,13 +45,6 @@ ktx sl [options] |------|-------------|---------| | `--connection-id ` | KTX connection id (required) | — | -### `sl write` - -| Flag | Description | Default | -|------|-------------|---------| -| `--connection-id ` | KTX connection id (required) | — | -| `--yaml ` | Semantic-layer source YAML content (required) | — | - ### `sl query` | Flag | Description | Default | @@ -82,20 +75,11 @@ ktx sl list --connection-id my-warehouse ktx sl list --json # Search sources as JSON -ktx sl list --json --query "revenue" - -# Read a source definition -ktx sl read orders --connection-id my-warehouse - -# Read a source definition as JSON -ktx sl read orders --connection-id my-warehouse --json +ktx sl search "revenue" --json # Validate a source against the live schema ktx sl validate orders --connection-id my-warehouse -# Write a new source from YAML -ktx sl write customers --connection-id my-warehouse --yaml "$(cat sources/customers.yaml)" - # Compile a query and view the generated SQL ktx sl query \ --connection-id my-warehouse \ @@ -159,5 +143,5 @@ Semantic-layer commands return human-readable output by default. Use `--json` or |-------|-------|----------| | Source not found | Source name or connection id is wrong | Run `ktx sl list --json` and retry with an exact source name and connection id | | Validation fails | YAML references missing columns, invalid joins, or invalid SQL expressions | Fix the source YAML and rerun `ktx sl validate` | -| Query compile fails | Measure, dimension, filter, or segment name is invalid | Read the source with `ktx sl read`, then retry using declared fields | +| Query compile fails | Measure, dimension, filter, or segment name is invalid | Search sources with `ktx sl search`, inspect the source YAML in your project files, then retry using declared fields | | Execution returns too many rows | `--max-rows` is missing or too high | Add `--max-rows` with a bounded value before executing | diff --git a/docs-site/content/docs/getting-started/quickstart.mdx b/docs-site/content/docs/getting-started/quickstart.mdx index 73e673dc..7aba00fd 100644 --- a/docs-site/content/docs/getting-started/quickstart.mdx +++ b/docs-site/content/docs/getting-started/quickstart.mdx @@ -208,9 +208,9 @@ KTX writes project state as plain files so agents can inspect and edit changes i |------|------------|---------| | `ktx.yaml` | `ktx setup` | Main project configuration: connections, LLM settings, embeddings, and context sources | | `.ktx/secrets/*` | `ktx setup` when file-backed secrets are selected | Local secret files referenced from `ktx.yaml`; do not commit these | -| `semantic-layer//*.yaml` | context build, ingestion, or `ktx sl write` | Semantic source definitions agents use for SQL generation | -| `wiki/global/*.md` | ingestion or `ktx wiki write --scope global` | Shared business context and metric definitions | -| `wiki/user//*.md` | `ktx wiki write --scope user` | User-scoped notes for one agent/user context | +| `semantic-layer//*.yaml` | context build, ingestion, or direct file edits | Semantic source definitions agents use for SQL generation | +| `wiki/global/*.md` | ingestion, memory capture, `ktx wiki write --scope global`, or direct file edits | Shared business context and metric definitions | +| `wiki/user//*.md` | memory capture, `ktx wiki write --scope user`, or direct file edits | User-scoped notes for one agent/user context | | `.claude/skills/ktx/SKILL.md`, `.agents/skills/ktx/SKILL.md` | CLI-mode agent integration setup | Agent instructions for calling public `ktx` commands | ## Verify it worked diff --git a/docs-site/content/docs/guides/serving-agents.mdx b/docs-site/content/docs/guides/serving-agents.mdx index 6dc45bc3..4a93ae43 100644 --- a/docs-site/content/docs/guides/serving-agents.mdx +++ b/docs-site/content/docs/guides/serving-agents.mdx @@ -26,10 +26,7 @@ ktx status --json # List sources ktx sl list --json ktx sl list --json --connection-id my-postgres -ktx sl list --json --query "revenue" - -# Read a source -ktx sl read orders --json --connection-id my-postgres +ktx sl search "revenue" --json # Run a query from a JSON file ktx sl query --json \ @@ -44,9 +41,6 @@ ktx sl query --json \ ```bash # Search wiki pages ktx wiki search "revenue recognition" --json --limit 10 - -# Read a specific page -ktx wiki read order-status-definitions --json ``` ## Setting Up Your Agent diff --git a/docs-site/content/docs/guides/writing-context.mdx b/docs-site/content/docs/guides/writing-context.mdx index 3819550a..b6ca3597 100644 --- a/docs-site/content/docs/guides/writing-context.mdx +++ b/docs-site/content/docs/guides/writing-context.mdx @@ -10,11 +10,11 @@ After building context through scanning and ingestion, you'll want to refine it Agents should refine context in this order: 1. `ktx sl list --json` — discover available sources and connection ids. -2. `ktx sl read --connection-id ` — inspect the current YAML. -3. Edit the source YAML directly or use `ktx sl write`. +2. `ktx sl search --json` — find source candidates for a concept. +3. Edit the source YAML directly in `semantic-layer//`. 4. `ktx sl validate --connection-id ` — verify columns, joins, and table references. 5. `ktx sl query ... --format sql` — compile a representative query without executing it. -6. `ktx wiki search ...` and `ktx wiki write ...` — add business context that does not belong in schema YAML. +6. `ktx wiki search ...` — check business context captured by ingest or memory. ## Semantic Sources @@ -33,13 +33,14 @@ ktx sl list --connection-id my-postgres ktx sl list --json ``` -### Reading a source +### Searching sources ```bash -ktx sl read orders --connection-id my-postgres +ktx sl search "revenue" --connection-id my-postgres --json ``` -This prints the full YAML definition for the source. +Search returns ranked source summaries. To inspect or edit a source, open the +YAML file under `semantic-layer//`. ### The source schema @@ -147,25 +148,10 @@ Column visibility controls what agents see: | `internal` | Available for joins and measures but not shown to agents | | `hidden` | Excluded entirely — useful for ETL columns | -### Writing a source +### Editing a source -```bash -ktx sl write orders --connection-id my-postgres --yaml ' -name: orders -table: public.orders -grain: [order_id] -columns: - - name: order_id - type: string - - name: total_amount - type: number -measures: - - name: total_revenue - expr: SUM(total_amount) -' -``` - -You can also edit source files directly — they live at `semantic-layer//.yaml` in your project directory. +Edit source files directly. They live at +`semantic-layer//.yaml` in your project directory. ### Validating sources @@ -225,11 +211,10 @@ The query planner is grain-aware — it understands the cardinality of joins and ### Workflow: edit and validate a source -1. `ktx sl read orders --connection-id my-postgres > /tmp/orders.yaml` — capture the current definition. -2. Edit `/tmp/orders.yaml` to add columns, measures, joins, or descriptions. -3. `ktx sl write orders --connection-id my-postgres --yaml "$(cat /tmp/orders.yaml)"` — write the updated source. -4. `ktx sl validate orders --connection-id my-postgres` — check the definition against the live schema. -5. `ktx sl query --connection-id my-postgres --measure total_revenue --dimension order_date --format sql` — compile a representative query. +1. Open `semantic-layer/my-postgres/orders.yaml`. +2. Edit the file to add columns, measures, joins, or descriptions. +3. `ktx sl validate orders --connection-id my-postgres` — check the definition against the live schema. +4. `ktx sl query --connection-id my-postgres --measure total_revenue --dimension order_date --format sql` — compile a representative query. If validation fails, fix the YAML before asking an agent to use the source. Common validation failures are missing columns, invalid join targets, and measure expressions that reference fields outside the source. @@ -260,42 +245,17 @@ wiki/ - **Global pages** apply across all connections — business definitions, metric standards, company terminology. - **User-scoped pages** are private to a user ID — personal notes, local gotchas, or context you do not want shared globally. -### Writing pages +### Editing pages -```bash -ktx wiki write order-status-definitions \ - --scope global \ - --summary "Business definitions for order status values" \ - --content "## Order Statuses - -- **pending**: Order placed but not yet processed -- **confirmed**: Payment received, awaiting fulfillment -- **shipped**: Order dispatched to carrier -- **delivered**: Order received by customer -- **cancelled**: Order cancelled before shipment - -Orders in pending status for more than 48 hours are flagged for review." \ - --tag orders \ - --tag definitions \ - --sl-ref orders -``` - -Write flags: - -| Flag | Description | -|------|-------------| -| `--scope ` | `global` (default) or `user` | -| `--summary ` | Short description for search results (required) | -| `--content ` | Full Markdown content (required) | -| `--tag ` | Categorization tag (repeatable) | -| `--ref ` | Reference to external resources (repeatable) | -| `--sl-ref ` | Link to a semantic source (repeatable) | +Create and edit wiki pages directly as Markdown files in the `wiki/` +directory, or with `ktx wiki write`. Ingest and memory capture also create +these pages automatically. Wiki page fields: | Field | Required | Description | |-------|----------|-------------| -| Key | Yes | Stable page identifier passed to `ktx wiki read` | +| Key | Yes | Stable page identifier used as the Markdown filename | | Summary | Yes | Short text shown in search results | | Content | Yes | Full Markdown business context | | Scope | No | `global` for shared context or `user` for user-scoped notes | @@ -303,20 +263,12 @@ Wiki page fields: | External refs | No | Links or identifiers for source-of-truth systems | | Semantic-layer refs | No | Source names the page explains or constrains | -You can also create and edit wiki pages directly as Markdown files in the `wiki/` directory. - ### Listing pages ```bash ktx wiki list ``` -### Reading a page - -```bash -ktx wiki read order-status-definitions -``` - ### Searching ```bash @@ -328,9 +280,9 @@ Search uses both full-text matching and semantic similarity — it finds relevan ### Workflow: add searchable business context 1. Search first: `ktx wiki search "order status definitions"`. -2. If no page already covers the rule, write a page with `ktx wiki write`. -3. Include a concise `--summary`; agents see this before loading full content. -4. Add `--tag` values for the business area and `--sl-ref` values for related semantic sources. +2. If no page already covers the rule, create or edit a Markdown file under `wiki/global/`. +3. Include concise frontmatter; agents see the summary before loading full content. +4. Add `tags` values for the business area and `sl_refs` values for related semantic sources. 5. Search again with the user's likely wording to confirm the page is discoverable. ## Common errors @@ -340,5 +292,5 @@ Search uses both full-text matching and semantic similarity — it finds relevan | `ktx sl validate` reports a missing column | YAML references a column that is absent from the scanned table | Run a fresh scan or update the YAML to match the warehouse schema | | Query compilation double-counts a measure | Join relationship or grain is missing or wrong | Add `grain` and explicit `relationship` values, then validate and recompile | | Agent cannot find a metric | Measure name or description does not match business terminology | Add a measure description and a wiki page with common synonyms | -| Knowledge search misses a page | Summary and tags do not include likely user wording | Rewrite the summary and add relevant tags, then search again | -| `ktx sl write` changes are hard to review | Large YAML was passed inline | Edit the source file directly or write from a temporary file, then review the git diff | +| Wiki search misses a page | Summary and tags do not include likely user wording | Rewrite the summary and add relevant tags, then search again | +| Semantic-layer changes are hard to review | The YAML edit is too large or unfocused | Split the change into smaller source-file edits, then review the git diff | diff --git a/docs-site/content/docs/integrations/agent-clients.mdx b/docs-site/content/docs/integrations/agent-clients.mdx index 4a982c34..95786f52 100644 --- a/docs-site/content/docs/integrations/agent-clients.mdx +++ b/docs-site/content/docs/integrations/agent-clients.mdx @@ -34,11 +34,9 @@ description: Use local KTX semantic context and wiki knowledge for this project. Available commands: - `ktx status --json --project-dir /path/to/project` - `ktx sl list --json --project-dir /path/to/project` -- `ktx sl list --json --project-dir /path/to/project --query ''` -- `ktx sl read '' --json --project-dir /path/to/project --connection-id ''` +- `ktx sl search '' --json --project-dir /path/to/project --connection-id ''` - `ktx sl query --json --project-dir /path/to/project --connection-id '' --query-file '' --execute --max-rows 100` - `ktx wiki search '' --json --project-dir /path/to/project --limit 10` -- `ktx wiki read '' --json --project-dir /path/to/project` ``` ### Workflow tips @@ -130,9 +128,7 @@ All supported agent clients call the same KTX CLI commands: | `ktx wiki read --json` | Read a wiki page | | `ktx wiki write ` | Write or update a wiki page | | `ktx sl list --json` | List semantic-layer sources | -| `ktx sl list --query --json` | Search semantic-layer sources | -| `ktx sl read --json --connection-id ` | Read a semantic source definition | -| `ktx sl write --connection-id ` | Write or update a semantic source | +| `ktx sl search --json` | Search semantic-layer sources | | `ktx sl validate --connection-id ` | Validate semantic source definitions | | `ktx sl query --json` | Execute a semantic-layer query when semantic compute is configured | diff --git a/packages/cli/src/commands/sl-commands.ts b/packages/cli/src/commands/sl-commands.ts index e1b985a3..d23674cd 100644 --- a/packages/cli/src/commands/sl-commands.ts +++ b/packages/cli/src/commands/sl-commands.ts @@ -41,7 +41,7 @@ async function runSlArgs(context: KtxCliCommandContext, args: KtxSlArgs): Promis export function registerSlCommands(program: Command, context: KtxCliCommandContext, commandName = 'sl'): void { const sl = program .command(commandName) - .description('List, read, validate, query, or write local semantic-layer sources') + .description('List, search, validate, or query local semantic-layer sources') .showHelpAfterError() .addHelpText( 'after', @@ -51,7 +51,31 @@ export function registerSlCommands(program: Command, context: KtxCliCommandConte sl.command('list') .description('List semantic-layer sources') .option('--connection-id ', 'KTX connection id') - .option('--query ', 'Search source names and descriptions') + .addOption( + new Option('--output ', 'Output mode: pretty (default in TTY), plain (TSV), or json').choices([ + 'pretty', + 'plain', + 'json', + ]), + ) + .option('--json', 'Shortcut for --output=json (overrides --output)', false) + .action( + async (options: { connectionId?: string; output?: 'pretty' | 'plain' | 'json'; json?: boolean }, command) => { + await runSlArgs(context, { + command: 'list', + projectDir: resolveCommandProjectDir(command), + connectionId: options.connectionId, + output: options.output, + json: options.json, + }); + }, + ); + + sl.command('search') + .description('Search semantic-layer sources') + .argument('', 'Search query') + .option('--connection-id ', 'KTX connection id') + .option('--limit ', 'Maximum search results', parsePositiveIntegerOption) .addOption( new Option('--output ', 'Output mode: pretty (default in TTY), plain (TSV), or json').choices([ 'pretty', @@ -62,35 +86,22 @@ export function registerSlCommands(program: Command, context: KtxCliCommandConte .option('--json', 'Shortcut for --output=json (overrides --output)', false) .action( async ( - options: { connectionId?: string; query?: string; output?: 'pretty' | 'plain' | 'json'; json?: boolean }, + query: string, + options: { connectionId?: string; limit?: number; output?: 'pretty' | 'plain' | 'json'; json?: boolean }, command, ) => { - await runSlArgs(context, { - command: 'list', - projectDir: resolveCommandProjectDir(command), - connectionId: options.connectionId, - query: options.query, - output: options.output, - json: options.json, - }); + await runSlArgs(context, { + command: 'search', + projectDir: resolveCommandProjectDir(command), + connectionId: options.connectionId, + query, + ...(options.limit !== undefined ? { limit: options.limit } : {}), + output: options.output, + json: options.json, + }); }, ); - sl.command('read') - .description('Read a semantic-layer source') - .argument('', 'Semantic-layer source name') - .requiredOption('--connection-id ', 'KTX connection id') - .option('--json', 'Print JSON output', false) - .action(async (sourceName: string, options: { connectionId: string; json?: boolean }, command) => { - await runSlArgs(context, { - command: 'read', - projectDir: resolveCommandProjectDir(command), - connectionId: options.connectionId, - sourceName, - json: options.json, - }); - }); - sl.command('validate') .description('Validate a semantic-layer source') .argument('', 'Semantic-layer source name') @@ -104,21 +115,6 @@ export function registerSlCommands(program: Command, context: KtxCliCommandConte }); }); - sl.command('write') - .description('Write a semantic-layer source') - .argument('', 'Semantic-layer source name') - .requiredOption('--connection-id ', 'KTX connection id') - .requiredOption('--yaml ', 'Semantic-layer source YAML') - .action(async (sourceName: string, options: { connectionId: string; yaml: string }, command) => { - await runSlArgs(context, { - command: 'write', - projectDir: resolveCommandProjectDir(command), - connectionId: options.connectionId, - sourceName, - yaml: options.yaml, - }); - }); - sl.command('query') .description('Compile or execute a semantic-layer query') .option('--connection-id ', 'KTX connection id') diff --git a/packages/cli/src/example-smoke.test.ts b/packages/cli/src/example-smoke.test.ts index 221c20f2..f1670544 100644 --- a/packages/cli/src/example-smoke.test.ts +++ b/packages/cli/src/example-smoke.test.ts @@ -79,12 +79,6 @@ describe('standalone local warehouse example', () => { parseJsonOutput<{ data: { items: Array<{ key: string; summary: string }> } }>(knowledgeList.stdout).data.items, ).toContainEqual(expect.objectContaining({ key: 'revenue', summary: 'Paid order value after refunds' })); - const knowledgeRead = await runBuiltCli(['wiki', 'read', 'revenue', '--json', '--project-dir', projectDir]); - expect(knowledgeRead).toMatchObject({ code: 0, stderr: '' }); - expect(parseJsonOutput<{ data: { content: string } }>(knowledgeRead.stdout).data.content).toContain( - 'Revenue is paid order amount after refund adjustments.', - ); - const slList = await runBuiltCli(['sl', 'list', '--json', '--project-dir', projectDir, '--connection-id', 'warehouse']); expect(slList).toMatchObject({ code: 0, stderr: '' }); expect( @@ -93,9 +87,9 @@ describe('standalone local warehouse example', () => { ).data.items, ).toContainEqual(expect.objectContaining({ connectionId: 'warehouse', name: 'orders', columnCount: 3 })); - const slRead = await runBuiltCli([ + const slSearch = await runBuiltCli([ 'sl', - 'read', + 'search', 'orders', '--json', '--connection-id', @@ -103,8 +97,10 @@ describe('standalone local warehouse example', () => { '--project-dir', projectDir, ]); - expect(slRead).toMatchObject({ code: 0, stderr: '' }); - expect(parseJsonOutput<{ data: { yaml: string } }>(slRead.stdout).data.yaml).toContain('name: orders'); + expect(slSearch).toMatchObject({ code: 0, stderr: '' }); + expect( + parseJsonOutput<{ data: { items: Array<{ connectionId: string; name: string }> } }>(slSearch.stdout).data.items, + ).toContainEqual(expect.objectContaining({ connectionId: 'warehouse', name: 'orders' })); const ingest = await runBuiltCli([ 'ingest', diff --git a/packages/cli/src/index.test.ts b/packages/cli/src/index.test.ts index 9064143a..d13c32c7 100644 --- a/packages/cli/src/index.test.ts +++ b/packages/cli/src/index.test.ts @@ -139,6 +139,112 @@ describe('runKtxCli', () => { expect(testIo.stderr()).toBe(''); }); + it('routes public wiki read and write commands', async () => { + const knowledge = vi.fn(async () => 0); + + const readIo = makeIo(); + await expect(runKtxCli(['--project-dir', tempDir, 'wiki', 'read', 'revenue', '--json'], readIo.io, { knowledge })) + .resolves.toBe(0); + expect(knowledge).toHaveBeenCalledWith( + { + command: 'read', + projectDir: tempDir, + key: 'revenue', + userId: 'local', + json: true, + }, + readIo.io, + ); + + const writeIo = makeIo(); + await expect( + runKtxCli( + [ + '--project-dir', + tempDir, + 'wiki', + 'write', + 'revenue', + '--scope', + 'user', + '--summary', + 'Revenue', + '--content', + 'Revenue.', + '--tag', + 'finance', + '--ref', + 'https://example.com/revenue', + '--sl-ref', + 'orders', + ], + writeIo.io, + { knowledge }, + ), + ).resolves.toBe(0); + expect(knowledge).toHaveBeenLastCalledWith( + { + command: 'write', + projectDir: tempDir, + key: 'revenue', + scope: 'USER', + userId: 'local', + summary: 'Revenue', + content: 'Revenue.', + tags: ['finance'], + refs: ['https://example.com/revenue'], + slRefs: ['orders'], + }, + writeIo.io, + ); + }); + + it('rejects removed public sl read/write commands', async () => { + const sl = vi.fn(async () => 0); + + for (const argv of [ + ['--project-dir', tempDir, 'sl', 'read', 'orders', '--connection-id', 'warehouse'], + ['--project-dir', tempDir, 'sl', 'write', 'orders', '--connection-id', 'warehouse', '--yaml', 'name: orders'], + ]) { + const io = makeIo(); + await expect(runKtxCli(argv, io.io, { sl })).resolves.toBe(1); + expect(io.stderr()).toMatch(/unknown command|error:/); + } + + expect(sl).not.toHaveBeenCalled(); + }); + + it('routes sl search and rejects the old sl list --query flag', async () => { + const sl = vi.fn(async () => 0); + + const searchIo = makeIo(); + await expect( + runKtxCli( + ['--project-dir', tempDir, 'sl', 'search', 'revenue', '--connection-id', 'warehouse', '--limit', '5', '--json'], + searchIo.io, + { sl }, + ), + ).resolves.toBe(0); + expect(sl).toHaveBeenCalledWith( + { + command: 'search', + projectDir: tempDir, + connectionId: 'warehouse', + query: 'revenue', + limit: 5, + json: true, + output: undefined, + }, + searchIo.io, + ); + + const listIo = makeIo(); + await expect( + runKtxCli(['--project-dir', tempDir, 'sl', 'list', '--query', 'revenue'], listIo.io, { sl }), + ).resolves.toBe(1); + expect(listIo.stderr()).toContain("unknown option '--query'"); + }); + it('routes runtime management commands with the CLI package version', async () => { const runtime = vi.fn(async () => 0); const installIo = makeIo(); diff --git a/packages/cli/src/setup-agents.ts b/packages/cli/src/setup-agents.ts index 3c7829c7..da9486f5 100644 --- a/packages/cli/src/setup-agents.ts +++ b/packages/cli/src/setup-agents.ts @@ -138,8 +138,7 @@ function cliInstructionContent(input: { projectDir: string; launcher: KtxCliLaun '', `- \`${ktxCommandLine(input.launcher, ['status', ...projectDirArgs])}\``, `- \`${ktxCommandLine(input.launcher, ['sl', 'list', ...projectDirArgs])}\``, - `- \`${ktxCommandLine(input.launcher, ['sl', 'list', ...projectDirArgs, '--query', ''])}\``, - `- \`${ktxCommandLine(input.launcher, ['sl', 'read', '', ...projectDirArgs, '--connection-id', ''])}\``, + `- \`${ktxCommandLine(input.launcher, ['sl', 'search', '', ...projectDirArgs, '--connection-id', ''])}\``, `- \`${ktxCommandLine(input.launcher, [ 'sl', 'query', @@ -153,7 +152,6 @@ function cliInstructionContent(input: { projectDir: string; launcher: KtxCliLaun '100', ])}\``, `- \`${ktxCommandLine(input.launcher, ['wiki', 'search', '', ...projectDirArgs, '--limit', '10'])}\``, - `- \`${ktxCommandLine(input.launcher, ['wiki', 'read', '', ...projectDirArgs])}\``, '', 'Use semantic-layer queries before direct database access. Do not print secrets or credential references.', '', diff --git a/packages/cli/src/sl.test.ts b/packages/cli/src/sl.test.ts index 8d360c58..48c7f4c7 100644 --- a/packages/cli/src/sl.test.ts +++ b/packages/cli/src/sl.test.ts @@ -38,6 +38,22 @@ function makeIo() { }; } +async function seedSlSource(input: { + projectDir: string; + connectionId?: string; + sourceName?: string; + yaml?: string; +}): Promise { + const project = await initKtxProject({ projectDir: input.projectDir, projectName: 'warehouse' }); + await project.fileStore.writeFile( + `semantic-layer/${input.connectionId ?? 'warehouse'}/${input.sourceName ?? 'orders'}.yaml`, + input.yaml ?? ORDERS_YAML, + 'ktx', + 'ktx@example.com', + 'Add semantic-layer source', + ); +} + describe('runKtxSl', () => { let tempDir: string; @@ -49,24 +65,9 @@ describe('runKtxSl', () => { await rm(tempDir, { recursive: true, force: true }); }); - it('writes, validates, reads, and lists semantic-layer sources', async () => { + it('validates, lists, and searches semantic-layer sources', async () => { const projectDir = join(tempDir, 'project'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); - - const writeIo = makeIo(); - await expect( - runKtxSl( - { - command: 'write', - projectDir, - connectionId: 'warehouse', - sourceName: 'orders', - yaml: ORDERS_YAML, - }, - writeIo.io, - ), - ).resolves.toBe(0); - expect(writeIo.stdout()).toContain('Wrote semantic-layer/warehouse/orders.yaml'); + await seedSlSource({ projectDir }); const validateIo = makeIo(); await expect( @@ -74,62 +75,49 @@ describe('runKtxSl', () => { ).resolves.toBe(0); expect(validateIo.stdout()).toContain('Valid semantic-layer source: warehouse/orders'); - const readIo = makeIo(); - await expect(runKtxSl({ command: 'read', projectDir, connectionId: 'warehouse', sourceName: 'orders' }, readIo.io)) - .resolves.toBe(0); - expect(readIo.stdout()).toContain('name: orders'); - const listIo = makeIo(); await expect(runKtxSl({ command: 'list', projectDir, connectionId: 'warehouse' }, listIo.io)).resolves.toBe(0); expect(listIo.stdout()).toContain('warehouse\torders\tcolumns=1\tmeasures=0\tjoins=0'); + + const searchIo = makeIo(); + await expect( + runKtxSl({ command: 'search', projectDir, connectionId: 'warehouse', query: 'order', json: true }, searchIo.io), + ).resolves.toBe(0); + expect(JSON.parse(searchIo.stdout())).toMatchObject({ + kind: 'list', + data: { + items: [ + expect.objectContaining({ + connectionId: 'warehouse', + name: 'orders', + score: expect.any(Number), + }), + ], + }, + meta: { command: 'sl search' }, + }); }); - it('prints semantic-layer reads and searched lists as public JSON envelopes', async () => { + it('prints semantic-layer list and search as public JSON envelopes', async () => { const projectDir = join(tempDir, 'project'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); - - await expect( - runKtxSl( - { - command: 'write', - projectDir, - connectionId: 'warehouse', - sourceName: 'orders', - yaml: [ - 'name: orders', - 'table: public.orders', - 'description: Paid order facts', - 'grain: [order_id]', - 'columns:', - ' - name: order_id', - ' type: string', - '', - ].join('\n'), - }, - makeIo().io, - ), - ).resolves.toBe(0); - - const readIo = makeIo(); - await expect( - runKtxSl( - { command: 'read', projectDir, connectionId: 'warehouse', sourceName: 'orders', json: true }, - readIo.io, - ), - ).resolves.toBe(0); - expect(JSON.parse(readIo.stdout())).toMatchObject({ - kind: 'sl.source', - data: { - connectionId: 'warehouse', - name: 'orders', - yaml: expect.stringContaining('name: orders'), - }, + await seedSlSource({ + projectDir, + yaml: [ + 'name: orders', + 'table: public.orders', + 'description: Paid order facts', + 'grain: [order_id]', + 'columns:', + ' - name: order_id', + ' type: string', + '', + ].join('\n'), }); const listIo = makeIo(); await expect( runKtxSl( - { command: 'list', projectDir, connectionId: 'warehouse', query: 'paid', json: true }, + { command: 'search', projectDir, connectionId: 'warehouse', query: 'paid', json: true }, listIo.io, ), ).resolves.toBe(0); @@ -145,7 +133,7 @@ describe('runKtxSl', () => { }), ], }, - meta: { command: 'sl list' }, + meta: { command: 'sl search' }, }); }); @@ -566,13 +554,7 @@ joins: [] it('emits sl list as a JSON envelope when output=json', async () => { const projectDir = join(tempDir, 'project'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); - - const writeIo = makeIo(); - await runKtxSl( - { command: 'write', projectDir, connectionId: 'warehouse', sourceName: 'orders', yaml: ORDERS_YAML }, - writeIo.io, - ); + await seedSlSource({ projectDir }); const listIo = makeIo(); const code = await runKtxSl( @@ -604,13 +586,7 @@ joins: [] it('emits sl list with grouping and Clack-style framing when output=pretty', async () => { const projectDir = join(tempDir, 'project'); - await initKtxProject({ projectDir, projectName: 'warehouse' }); - - const writeIo = makeIo(); - await runKtxSl( - { command: 'write', projectDir, connectionId: 'warehouse', sourceName: 'orders', yaml: ORDERS_YAML }, - writeIo.io, - ); + await seedSlSource({ projectDir }); const listIo = makeIo(); const code = await runKtxSl( diff --git a/packages/cli/src/sl.ts b/packages/cli/src/sl.ts index ebf3eca7..baff239b 100644 --- a/packages/cli/src/sl.ts +++ b/packages/cli/src/sl.ts @@ -13,10 +13,10 @@ import { readLocalSlSource, searchLocalSlSources, validateLocalSlSource, - writeLocalSlSource, + type LocalSlSourceSearchResult, + type LocalSlSourceSummary, type SemanticLayerQueryInput, } from '@ktx/context/sl'; -import { writeJsonResult } from './io/print-list.js'; import { createManagedPythonSemanticLayerComputePort, type KtxManagedPythonInstallPolicy, @@ -28,10 +28,17 @@ profileMark('module:sl'); type SlQueryFormat = 'json' | 'sql'; export type KtxSlArgs = - | { command: 'list'; projectDir: string; connectionId?: string; query?: string; output?: string; json?: boolean } - | { command: 'read'; projectDir: string; connectionId: string; sourceName: string; json?: boolean } + | { command: 'list'; projectDir: string; connectionId?: string; output?: string; json?: boolean } + | { + command: 'search'; + projectDir: string; + connectionId?: string; + query: string; + limit?: number; + output?: string; + json?: boolean; + } | { command: 'validate'; projectDir: string; connectionId: string; sourceName: string } - | { command: 'write'; projectDir: string; connectionId: string; sourceName: string; yaml: string } | { command: 'query'; projectDir: string; @@ -73,6 +80,35 @@ function slSearchEmbeddingService(project: KtxLocalProject, deps: KtxSlDeps): Kt return provider ? new KtxIngestEmbeddingPortAdapter(provider) : null; } +async function printSlSources(input: { + rows: ReadonlyArray; + command: 'sl list' | 'sl search'; + output?: string; + json?: boolean; + io: KtxSlIo; + emptyMessage: string; +}): Promise { + const { resolveOutputMode } = await import('./io/mode.js'); + const { printList } = await import('./io/print-list.js'); + const mode = resolveOutputMode({ explicit: input.output, json: input.json, io: input.io }); + printList({ + rows: input.rows, + columns: [ + { key: 'connectionId', label: 'CONNECTION', plain: '' }, + { key: 'name', label: 'NAME', plain: '' }, + { key: 'columnCount', label: 'COLS', plain: 'columns=', dim: true }, + { key: 'measureCount', label: 'MEASURES', plain: 'measures=', dim: true }, + { key: 'joinCount', label: 'JOINS', plain: 'joins=', dim: true }, + { key: 'description', label: 'DESCRIPTION', plain: false, optional: true, dim: true }, + ], + groupBy: 'connectionId', + emptyMessage: input.emptyMessage, + command: input.command, + mode, + io: input.io, + }); +} + async function readSlQueryFile(path: string): Promise { const parsed = JSON.parse(await readFile(path, 'utf-8')) as unknown; if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) { @@ -85,51 +121,32 @@ export async function runKtxSl(args: KtxSlArgs, io: KtxSlIo = process, deps: Ktx try { const project = await (deps.loadProject ?? loadKtxProject)({ projectDir: args.projectDir }); if (args.command === 'list') { - const sources = args.query - ? await searchLocalSlSources(project, { - connectionId: args.connectionId, - query: args.query, - embeddingService: slSearchEmbeddingService(project, deps), - }) - : await listLocalSlSources(project, { connectionId: args.connectionId }); - const { resolveOutputMode } = await import('./io/mode.js'); - const { printList } = await import('./io/print-list.js'); - const mode = resolveOutputMode({ explicit: args.output, json: args.json, io }); - printList({ + const sources = await listLocalSlSources(project, { connectionId: args.connectionId }); + await printSlSources({ rows: sources, - columns: [ - { key: 'connectionId', label: 'CONNECTION', plain: '' }, - { key: 'name', label: 'NAME', plain: '' }, - { key: 'columnCount', label: 'COLS', plain: 'columns=', dim: true }, - { key: 'measureCount', label: 'MEASURES', plain: 'measures=', dim: true }, - { key: 'joinCount', label: 'JOINS', plain: 'joins=', dim: true }, - { key: 'description', label: 'DESCRIPTION', plain: false, optional: true, dim: true }, - ], - groupBy: 'connectionId', emptyMessage: `No semantic-layer sources found in ${project.projectDir}`, command: 'sl list', - mode, + output: args.output, + json: args.json, io, }); return 0; } - if (args.command === 'read') { - const source = await readLocalSlSource(project, { + if (args.command === 'search') { + const sources = await searchLocalSlSources(project, { connectionId: args.connectionId, - sourceName: args.sourceName, + query: args.query, + embeddingService: slSearchEmbeddingService(project, deps), + limit: args.limit, + }); + await printSlSources({ + rows: sources, + emptyMessage: `No semantic-layer sources matched "${args.query}" in ${project.projectDir}`, + command: 'sl search', + output: args.output, + json: args.json, + io, }); - if (!source) { - throw new Error(`Semantic-layer source "${args.connectionId}/${args.sourceName}" was not found`); - } - if (args.json) { - writeJsonResult(io, { - kind: 'sl.source', - data: source, - meta: { command: 'sl read' }, - }); - return 0; - } - io.stdout.write(source.yaml); return 0; } if (args.command === 'validate') { @@ -178,14 +195,8 @@ export async function runKtxSl(args: KtxSlArgs, io: KtxSlIo = process, deps: Ktx io.stdout.write(`${JSON.stringify(result, null, 2)}\n`); return 0; } - - const write = await writeLocalSlSource(project, { - connectionId: args.connectionId, - sourceName: args.sourceName, - yaml: args.yaml, - }); - io.stdout.write(`Wrote ${write.path}\n`); - return 0; + const _exhaustive: never = args; + throw new Error(`Unsupported sl command: ${JSON.stringify(_exhaustive)}`); } catch (error) { io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); return 1; diff --git a/scripts/examples-docs.test.mjs b/scripts/examples-docs.test.mjs index 793566ed..62a25cf7 100644 --- a/scripts/examples-docs.test.mjs +++ b/scripts/examples-docs.test.mjs @@ -154,10 +154,9 @@ describe('standalone example docs', () => { for (const command of [ 'ktx status --json', 'ktx sl list --json', - 'ktx sl read orders --json', + 'ktx sl search "revenue" --json', 'ktx sl query --json', 'ktx wiki search "revenue recognition" --json', - 'ktx wiki read order-status-definitions --json', ]) { assert.match(servingAgents, new RegExp(command.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))); } diff --git a/scripts/package-artifacts.mjs b/scripts/package-artifacts.mjs index 24e9ab45..b74b4277 100644 --- a/scripts/package-artifacts.mjs +++ b/scripts/package-artifacts.mjs @@ -729,23 +729,22 @@ try { 'exec', 'ktx', 'sl', - 'list', + 'search', + 'orders', '--json', '--connection-id', 'warehouse', - '--query', - 'orders', '--project-dir', projectDir, ]); - const slSearchJson = parseJsonResult('ktx sl list', slSearch); + const slSearchJson = parseJsonResult('ktx sl search', slSearch); assert.equal(slSearchJson.kind, 'list'); assert.equal(slSearchJson.data.items.length, 1); assert.equal(slSearchJson.data.items[0].connectionId, 'warehouse'); assert.equal(slSearchJson.data.items[0].name, 'orders'); assert.equal(typeof slSearchJson.data.items[0].score, 'number'); requireIncludes(slSearchJson.data.items[0].matchReasons, 'lexical', 'sl search match reasons'); - process.stdout.write('ktx sl list hybrid metadata verified\\n'); + process.stdout.write('ktx sl search hybrid metadata verified\\n'); const slQuery = await run('pnpm', ['exec', 'ktx', 'sl', 'query', '--connection-id', diff --git a/scripts/package-artifacts.test.mjs b/scripts/package-artifacts.test.mjs index 1235b147..06671d7c 100644 --- a/scripts/package-artifacts.test.mjs +++ b/scripts/package-artifacts.test.mjs @@ -459,7 +459,7 @@ describe('verification snippets', () => { assert.match(source, /wiki', 'global', 'revenue\.md'/); assert.match(source, /run\('pnpm', \[\s*'exec',\s*'ktx',\s*'wiki',\s*'search'/); assert.match(source, /semantic-layer', 'warehouse', 'orders\.yaml'/); - assert.match(source, /run\('pnpm', \[\s*'exec',\s*'ktx',\s*'sl',\s*'list'/); + assert.match(source, /run\('pnpm', \[\s*'exec',\s*'ktx',\s*'sl',\s*'search',\s*'orders'/); assert.match(source, /orders\.order_count/); assert.match(source, /node:sqlite/); assert.match(source, /driver: sqlite/);