From 97da9919e9f8c1f761d09017027b4563c8c58441 Mon Sep 17 00:00:00 2001 From: Andrey Avtomonov Date: Wed, 13 May 2026 15:55:00 +0200 Subject: [PATCH 1/2] refactor: remove legacy compatibility paths (#64) * refactor: remove legacy compatibility paths * fix: support legacy metabase native queries * test: use canonical semantic layer descriptions * Rename CLI description * Recover setup scan from SQLite ABI mismatch * Remove legacy product name from CLI help --- .../semantic-layer/warehouse/orders.yaml | 3 +- .../global/orbit-customers-source.md | 2 +- .../global/sales-ops-cs-handoff-process.md | 2 +- packages/cli/src/cli-program.ts | 2 +- packages/cli/src/commands/setup-commands.ts | 12 +- packages/cli/src/index.test.ts | 4 +- packages/cli/src/ingest.test-utils.ts | 12 +- packages/cli/src/ingest.test.ts | 3 +- packages/cli/src/local-adapters.ts | 2 - .../cli/src/local-scan-connectors.test.ts | 9 +- packages/cli/src/local-scan-connectors.ts | 2 +- packages/cli/src/next-steps.test.ts | 25 ---- packages/cli/src/public-ingest.ts | 2 +- packages/cli/src/setup-databases.test.ts | 62 ++++++++- packages/cli/src/setup-databases.ts | 121 ++++++++++++++++-- packages/cli/src/setup-sources.ts | 10 +- packages/cli/src/setup.ts | 2 - packages/cli/src/sl.test.ts | 2 +- .../memory_agent_bundle_ingest_reconcile.md | 4 +- .../context/skills/ingest_triage/SKILL.md | 4 +- .../context/skills/metabase_ingest/SKILL.md | 2 +- .../context/skills/metricflow_ingest/SKILL.md | 6 +- packages/context/skills/sl/SKILL.md | 3 +- packages/context/skills/sl_capture/SKILL.md | 11 +- .../historic-sql/historic-sql.adapter.test.ts | 4 +- .../historic-sql/historic-sql.adapter.ts | 7 - .../adapters/historic-sql/projection.test.ts | 17 ++- .../adapters/historic-sql/projection.ts | 14 +- .../adapters/historic-sql/types.test.ts | 12 +- .../src/ingest/adapters/historic-sql/types.ts | 24 +--- .../adapters/looker/local-looker.adapter.ts | 8 +- .../adapters/metabase/client-port.test.ts | 11 +- .../ingest/adapters/metabase/client-port.ts | 2 +- .../ingest/adapters/metabase/client.test.ts | 41 +++++- .../src/ingest/adapters/metabase/client.ts | 30 +---- .../metabase/local-metabase.adapter.test.ts | 13 -- .../metabase/local-metabase.adapter.ts | 6 +- .../metricflow/import-semantic-models.ts | 31 ----- .../src/ingest/ingest-bundle.runner.test.ts | 3 +- .../context/src/ingest/local-adapters.test.ts | 12 +- packages/context/src/ingest/local-adapters.ts | 24 ++-- .../src/ingest/report-snapshot.test.ts | 9 +- .../context/src/ingest/report-snapshot.ts | 25 +--- packages/context/src/ingest/reports.ts | 3 +- .../src/ingest/stages/stage-index.types.ts | 2 +- .../tools/emit-eviction-decision.tool.ts | 2 +- .../emit-reconciliation-records.tool.test.ts | 8 +- .../src/ingest/tools/eviction-list.tool.ts | 2 +- .../warehouse-catalog.service.ts | 16 +-- .../src/ingest/wiki-sl-ref-repair.test.ts | 2 +- .../src/mcp/local-project-ports.test.ts | 3 +- .../context/src/memory/capture-signals.ts | 3 +- packages/context/src/memory/types.ts | 2 +- packages/context/src/project/config.test.ts | 38 +++--- packages/context/src/project/config.ts | 39 ++---- .../src/scan/relationship-benchmarks.test.ts | 4 +- .../src/scan/relationship-diagnostics.test.ts | 2 +- .../src/search/backend-conformance.test.ts | 3 +- .../src/sl/description-normalization.ts | 5 - packages/context/src/sl/local-sl.test.ts | 6 +- .../src/sl/pglite-sl-search-prototype.test.ts | 9 +- packages/context/src/sl/schemas.ts | 6 +- .../src/sl/semantic-layer.service.test.ts | 12 +- .../context/src/sl/semantic-layer.service.ts | 42 +----- .../src/sl/tools/sl-edit-source.tool.ts | 2 +- .../src/sl/tools/sl-warehouse-validation.ts | 2 +- .../src/sl/tools/sl-write-source.tool.test.ts | 6 +- .../src/sl/tools/sl-write-source.tool.ts | 2 +- packages/context/src/tools/tool-session.ts | 2 +- .../context/src/wiki/local-knowledge.test.ts | 25 ---- .../data.sqlite | Bin .../expected-links.yaml | 0 .../fixture.yaml | 4 +- .../snapshot.json | 2 +- python/ktx-sl/semantic_layer/loader.py | 11 +- python/ktx-sl/semantic_layer/manifest.py | 47 ++----- .../ktx-sl/sources/b2b_saas/churn_risk.yaml | 13 +- .../ktx-sl/sources/ecommerce/churn_risk.yaml | 7 +- python/ktx-sl/tests/test_manifest.py | 20 +-- ...d-evidence-fusion-adversarial-fixtures.mjs | 8 +- scripts/check-boundaries.mjs | 6 +- scripts/check-boundaries.test.mjs | 2 +- scripts/examples-docs.test.mjs | 1 - 83 files changed, 442 insertions(+), 527 deletions(-) rename packages/context/test/fixtures/relationship-benchmarks/{abbreviated_legacy_no_declared_constraints => abbreviated_old_no_declared_constraints}/data.sqlite (100%) rename packages/context/test/fixtures/relationship-benchmarks/{abbreviated_legacy_no_declared_constraints => abbreviated_old_no_declared_constraints}/expected-links.yaml (100%) rename packages/context/test/fixtures/relationship-benchmarks/{abbreviated_legacy_no_declared_constraints => abbreviated_old_no_declared_constraints}/fixture.yaml (50%) rename packages/context/test/fixtures/relationship-benchmarks/{abbreviated_legacy_no_declared_constraints => abbreviated_old_no_declared_constraints}/snapshot.json (98%) diff --git a/examples/local-warehouse/semantic-layer/warehouse/orders.yaml b/examples/local-warehouse/semantic-layer/warehouse/orders.yaml index ffcca12b..8ffbe973 100644 --- a/examples/local-warehouse/semantic-layer/warehouse/orders.yaml +++ b/examples/local-warehouse/semantic-layer/warehouse/orders.yaml @@ -1,6 +1,7 @@ name: orders table: public.orders -description: Orders placed through the storefront. +descriptions: + user: Orders placed through the storefront. grain: - id columns: diff --git a/packages/cli/assets/demo/orbit/knowledge/global/orbit-customers-source.md b/packages/cli/assets/demo/orbit/knowledge/global/orbit-customers-source.md index e98c1663..2c9f2c65 100644 --- a/packages/cli/assets/demo/orbit/knowledge/global/orbit-customers-source.md +++ b/packages/cli/assets/demo/orbit/knowledge/global/orbit-customers-source.md @@ -57,4 +57,4 @@ Always join through `customer.id`. Do not join on `email`. - **Join key:** Always use `customer.id`, never `email`. - **Timezone:** `created_at` and `last_seen_at` are UTC. Confirm whether a question expects UTC or a local business day before filtering. - **Paying vs. all:** `free` customers must be excluded from paying-customer follow-ups. Use `paying_customer_count`, not `customer_count`. -- **plan_tier values:** `free`, `pro`, `enterprise`. Note: `pro_plus` is a legacy alias for `growth` in the account/contract layer (see `orbit-plan-segment-normalization`), but `plan_tier` on this table uses `pro` not `pro_plus`. +- **plan_tier values:** `free`, `pro`, `enterprise`. Note: use the canonical plan names from the account/contract layer (see `orbit-plan-segment-normalization`); `plan_tier` on this table uses `pro` rather than `growth`. diff --git a/packages/cli/assets/demo/orbit/knowledge/global/sales-ops-cs-handoff-process.md b/packages/cli/assets/demo/orbit/knowledge/global/sales-ops-cs-handoff-process.md index d547d026..65693ee6 100644 --- a/packages/cli/assets/demo/orbit/knowledge/global/sales-ops-cs-handoff-process.md +++ b/packages/cli/assets/demo/orbit/knowledge/global/sales-ops-cs-handoff-process.md @@ -27,7 +27,7 @@ Sales Ops must complete the handoff **before the first implementation call**. Cu | Field | Notes | |---|---| -| Current plan | Starter / Growth / Enterprise — use canonical plan name, not legacy aliases | +| Current plan | Starter / Growth / Enterprise — use canonical plan name | | Account segment | self_serve / commercial / enterprise (see `orbit-plan-segment-normalization`) | | Contract shape | Term, ARR, any discounts or custom terms | | Renewal contact | Named person on the customer side responsible for renewal | diff --git a/packages/cli/src/cli-program.ts b/packages/cli/src/cli-program.ts index dbe73a72..69437aec 100644 --- a/packages/cli/src/cli-program.ts +++ b/packages/cli/src/cli-program.ts @@ -216,7 +216,7 @@ export function resolveCommandProjectDirOverride(command: CommandWithGlobalOptio function createBaseProgram(info: KtxCliPackageInfo, io: KtxCliIo): Command { return new Command() .name('ktx') - .description('Standalone KTX developer CLI') + .description('KTX data agent context layer CLI') .option('--project-dir ', 'KTX project directory (default: KTX_PROJECT_DIR, nearest ktx.yaml, or cwd)') .option('--debug', 'Enable diagnostic logging to stderr') .version(`${info.name} ${info.version}`, '-v, --version', 'Show CLI version') diff --git a/packages/cli/src/commands/setup-commands.ts b/packages/cli/src/commands/setup-commands.ts index 6a215651..1688724d 100644 --- a/packages/cli/src/commands/setup-commands.ts +++ b/packages/cli/src/commands/setup-commands.ts @@ -121,7 +121,6 @@ function shouldShowSetupEntryMenu( disableHistoricSql?: boolean; historicSqlWindowDays?: number; historicSqlMinExecutions?: number; - historicSqlMinCalls?: number; historicSqlServiceAccountPattern?: string[]; historicSqlRedactionPattern?: string[]; skipDatabases?: boolean; @@ -194,7 +193,6 @@ function shouldShowSetupEntryMenu( 'disableHistoricSql', 'historicSqlWindowDays', 'historicSqlMinExecutions', - 'historicSqlMinCalls', 'skipDatabases', 'source', 'sourceConnectionId', @@ -283,11 +281,6 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo .option('--disable-historic-sql', 'Disable Historic SQL for the selected database', false) .option('--historic-sql-window-days ', 'Historic SQL query-history window', positiveInteger) .option('--historic-sql-min-executions ', 'Minimum Historic SQL executions for a template', positiveInteger) - .option( - '--historic-sql-min-calls ', - 'Alias for --historic-sql-min-executions', - positiveInteger, - ) .option( '--historic-sql-service-account-pattern ', 'Historic SQL service-account regex; repeatable', @@ -379,7 +372,6 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo const mode = options.new ? 'new' : options.existing ? 'existing' : 'auto'; const resolvedAgentScope = options.global ? 'global' : options.agentScope; - const historicSqlMinExecutions = options.historicSqlMinExecutions ?? options.historicSqlMinCalls; await runSetupArgs(context, { command: 'run', projectDir: resolveCommandProjectDir(command), @@ -410,7 +402,9 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo ...(options.enableHistoricSql ? { enableHistoricSql: true } : {}), ...(options.disableHistoricSql ? { disableHistoricSql: true } : {}), ...(options.historicSqlWindowDays !== undefined ? { historicSqlWindowDays: options.historicSqlWindowDays } : {}), - ...(historicSqlMinExecutions !== undefined ? { historicSqlMinExecutions } : {}), + ...(options.historicSqlMinExecutions !== undefined + ? { historicSqlMinExecutions: options.historicSqlMinExecutions } + : {}), ...(options.historicSqlServiceAccountPattern.length > 0 ? { historicSqlServiceAccountPatterns: options.historicSqlServiceAccountPattern } : {}), diff --git a/packages/cli/src/index.test.ts b/packages/cli/src/index.test.ts index 817653f6..f41f4b6a 100644 --- a/packages/cli/src/index.test.ts +++ b/packages/cli/src/index.test.ts @@ -123,12 +123,12 @@ describe('runKtxCli', () => { await expect(runKtxCli(['--help'], testIo.io)).resolves.toBe(0); expect(testIo.stdout()).toContain('Usage: ktx [options] [command]'); + expect(testIo.stdout()).toContain('KTX data agent context layer CLI'); for (const command of ['setup', 'connection', 'ingest', 'wiki', 'sl', 'status', 'scan']) { expect(testIo.stdout()).toContain(`${command}`); } for (const removed of ['demo', 'init', 'connect', 'ask', 'knowledge', 'agent', 'completion', 'serve']) { - expect(testIo.stdout()).not.toContain(`${removed} [`); - expect(testIo.stdout()).not.toContain(`${removed} `); + expect(testIo.stdout()).not.toMatch(new RegExp(`^\\s+${removed}(?:\\s|\\[|$)`, 'm')); } expect(testIo.stdout()).toContain('--project-dir '); expect(testIo.stdout()).toContain('KTX_PROJECT_DIR'); diff --git a/packages/cli/src/ingest.test-utils.ts b/packages/cli/src/ingest.test-utils.ts index 3596d215..73190b0d 100644 --- a/packages/cli/src/ingest.test-utils.ts +++ b/packages/cli/src/ingest.test-utils.ts @@ -376,7 +376,7 @@ const SYNC_MODE_METABASE_CARDS: MetabaseCard[] = [ collection_id: 12, archived: false, result_metadata: [], - dataset_query: { type: 'native', database: 1, native: { query: 'select 101 as id' } }, + dataset_query: { type: 'native', database: 1, stages: [{ 'lib/type': 'mbql.stage/native', native: 'select 101 as id' }] }, parameters: [], dashboard_count: 0, }, @@ -390,7 +390,7 @@ const SYNC_MODE_METABASE_CARDS: MetabaseCard[] = [ collection_id: 12, archived: false, result_metadata: [], - dataset_query: { type: 'native', database: 1, native: { query: 'select 102 as id' } }, + dataset_query: { type: 'native', database: 1, stages: [{ 'lib/type': 'mbql.stage/native', native: 'select 102 as id' }] }, parameters: [], dashboard_count: 0, }, @@ -404,7 +404,7 @@ const SYNC_MODE_METABASE_CARDS: MetabaseCard[] = [ collection_id: 13, archived: false, result_metadata: [], - dataset_query: { type: 'native', database: 1, native: { query: 'select 103 as id' } }, + dataset_query: { type: 'native', database: 1, stages: [{ 'lib/type': 'mbql.stage/native', native: 'select 103 as id' }] }, parameters: [], dashboard_count: 0, }, @@ -454,11 +454,11 @@ function createSyncModeMetabaseClient(): MetabaseRuntimeClient { }, getAllCards: async () => SYNC_MODE_METABASE_CARDS.map(metabaseCardSummary), convertMbqlToNative: async () => ({ query: 'select 1' }), - getNativeSql: (card) => card.dataset_query?.native?.query ?? null, + getNativeSql: (card) => card.dataset_query?.stages?.[0]?.native ?? null, getTemplateTags: () => ({}), - getCardSql: async (card) => card.dataset_query?.native?.query ?? null, + getCardSql: async (card) => card.dataset_query?.stages?.[0]?.native ?? null, getResolvedSql: async (card) => ({ - resolvedSql: card.dataset_query?.native?.query ?? `select ${card.id} as id`, + resolvedSql: card.dataset_query?.stages?.[0]?.native ?? `select ${card.id} as id`, templateTags: [], resolutionStatus: 'resolved', }), diff --git a/packages/cli/src/ingest.test.ts b/packages/cli/src/ingest.test.ts index 24f8c1ca..c847d53a 100644 --- a/packages/cli/src/ingest.test.ts +++ b/packages/cli/src/ingest.test.ts @@ -705,7 +705,6 @@ describe('runKtxIngest', () => { patternPagesWritten: 30, stalePatternPagesMarked: 2, archivedPatternPages: 3, - legacyPagesDeleted: 4, }, errors: [], warnings: [], @@ -739,7 +738,7 @@ describe('runKtxIngest', () => { expect(io.stderr()).toBe(''); expect(io.stdout()).toContain('Adapter: historic-sql\n'); - expect(io.stdout()).toContain('Saved memory: 39 wiki, 57 SL\n'); + expect(io.stdout()).toContain('Saved memory: 35 wiki, 57 SL\n'); }); it('returns a non-zero code when local ingest reports failed work units', async () => { diff --git a/packages/cli/src/local-adapters.ts b/packages/cli/src/local-adapters.ts index 8557674c..9a6915c2 100644 --- a/packages/cli/src/local-adapters.ts +++ b/packages/cli/src/local-adapters.ts @@ -1,4 +1,3 @@ -import { join } from 'node:path'; import { createBigQueryLiveDatabaseIntrospection, isKtxBigQueryConnectionConfig, @@ -298,7 +297,6 @@ function historicSqlOptionsForLocalRun(project: KtxLocalProject, options: KtxCli const base = { sqlAnalysis: ktxCliHistoricSqlAnalysis(options), - postgresBaselineRootDir: join(project.projectDir, '.ktx/cache/historic-sql'), }; if (dialect === 'postgres') { diff --git a/packages/cli/src/local-scan-connectors.test.ts b/packages/cli/src/local-scan-connectors.test.ts index 0fe57518..087e978d 100644 --- a/packages/cli/src/local-scan-connectors.test.ts +++ b/packages/cli/src/local-scan-connectors.test.ts @@ -62,10 +62,7 @@ describe('createKtxCliScanConnector', () => { expect(connector.driver).toBe('sqlite'); }); - it.each([ - ['maxBytesBilled', ' maxBytesBilled: 123456789', 123456789], - ['max_bytes_billed', ' max_bytes_billed: "987654321"', '987654321'], - ])('passes BigQuery %s from standalone config', async (_label, byteCapLine, expectedMaxBytesBilled) => { + it('passes BigQuery max_bytes_billed from standalone config', async () => { await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); await writeFile( join(tempDir, 'ktx.yaml'), @@ -76,7 +73,7 @@ describe('createKtxCliScanConnector', () => { ' driver: bigquery', ' dataset_id: analytics', ' readonly: true', - byteCapLine, + ' max_bytes_billed: "987654321"', '', ].join('\n'), 'utf-8', @@ -90,7 +87,7 @@ describe('createKtxCliScanConnector', () => { expect(bigQueryMock.constructorInputs).toEqual([ expect.objectContaining({ connectionId: 'warehouse', - maxBytesBilled: expectedMaxBytesBilled, + maxBytesBilled: '987654321', }), ]); }); diff --git a/packages/cli/src/local-scan-connectors.ts b/packages/cli/src/local-scan-connectors.ts index d3377b0c..3058b96e 100644 --- a/packages/cli/src/local-scan-connectors.ts +++ b/packages/cli/src/local-scan-connectors.ts @@ -6,7 +6,7 @@ const SUPPORTED_DRIVERS = 'sqlite, postgres, mysql, clickhouse, sqlserver, bigqu function bigQueryMaxBytesBilled( connection: KtxLocalProject['config']['connections'][string], ): number | string | undefined { - const raw = connection.maxBytesBilled ?? connection.max_bytes_billed; + const raw = connection.max_bytes_billed; if (typeof raw === 'number') { return Number.isFinite(raw) && raw > 0 ? raw : undefined; } diff --git a/packages/cli/src/next-steps.test.ts b/packages/cli/src/next-steps.test.ts index b4706d72..facb4eb8 100644 --- a/packages/cli/src/next-steps.test.ts +++ b/packages/cli/src/next-steps.test.ts @@ -6,8 +6,6 @@ import { formatSetupNextStepLines, } from './next-steps.js'; -const command = (...parts: string[]) => parts.join(' '); - describe('KTX demo next steps', () => { it('uses supported context-build commands before agent usage', () => { expect(KTX_CONTEXT_BUILD_COMMANDS).toEqual([ @@ -57,29 +55,6 @@ describe('KTX demo next steps', () => { expect(rendered).not.toContain('Optional MCP:'); }); - it('does not advertise removed Commander migration commands', () => { - const rendered = formatNextStepLines().join('\n'); - - expect(rendered).toContain('ktx status --json'); - expect(rendered).not.toContain('ktx agent'); - expect(rendered).toContain('ktx sl list'); - expect(rendered).toContain('ktx wiki list'); - - for (const removed of [ - command('ktx', 'ask'), - command('ktx', 'mcp'), - command('ktx', 'connect'), - command('ktx', 'knowledge'), - command('dev', 'model'), - command('dev', 'knowledge'), - command('ktx', 'ingest', 'run'), - command('ktx', 'ingest', 'replay'), - command('ktx', 'serve', '--mcp', 'stdio', '--user-id', 'local'), - ]) { - expect(rendered).not.toContain(removed); - } - }); - it('keeps setup next steps focused on building context when the build is not ready', () => { const rendered = formatSetupNextStepLines({ setupReady: true, diff --git a/packages/cli/src/public-ingest.ts b/packages/cli/src/public-ingest.ts index f8296177..71d93e4a 100644 --- a/packages/cli/src/public-ingest.ts +++ b/packages/cli/src/public-ingest.ts @@ -92,7 +92,7 @@ function normalizedDriver(connection: KtxProjectConnectionConfig): string { } function sourceDirForConnection(connection: KtxProjectConnectionConfig): string | undefined { - const value = connection.source_dir ?? connection.sourceDir; + const value = connection.source_dir; return typeof value === 'string' && value.trim().length > 0 ? value.trim() : undefined; } diff --git a/packages/cli/src/setup-databases.test.ts b/packages/cli/src/setup-databases.test.ts index 2999d365..95d1e3fb 100644 --- a/packages/cli/src/setup-databases.test.ts +++ b/packages/cli/src/setup-databases.test.ts @@ -64,8 +64,6 @@ function textInputPrompt(message: string): string { return `${title}\n│\n│ ${bodyLines.join('\n│ ')}\n│ Press Escape to go back.\n│`; } -const legacyHistoricSqlServiceAccountPatternsKey = ['serviceAccount', 'UserPatterns'].join(''); - describe('setup databases step', () => { let tempDir: string; @@ -1255,6 +1253,7 @@ describe('setup databases step', () => { io.io, { testConnection: vi.fn(async () => 0), + rebuildNativeSqlite: vi.fn(async () => 1), scanConnection: vi.fn(async (_projectDir: string, _connectionId: string, commandIo: KtxCliIo) => { commandIo.stderr.write( [ @@ -1280,6 +1279,60 @@ describe('setup databases step', () => { expect(io.stderr()).not.toMatch(/^Native SQLite is built for a different Node.js ABI\./m); }); + it('rebuilds native SQLite once and retries setup scanning after a Node ABI mismatch', async () => { + const io = makeIo(); + const scanConnection = vi.fn(async (_projectDir: string, _connectionId: string, commandIo: KtxCliIo) => { + if (scanConnection.mock.calls.length === 1) { + commandIo.stderr.write( + [ + "The module '/workspace/node_modules/better-sqlite3/build/Release/better_sqlite3.node'", + 'was compiled against a different Node.js version using', + 'NODE_MODULE_VERSION 147. This version of Node.js requires', + 'NODE_MODULE_VERSION 137. Please try re-compiling or re-installing', + 'the module (for instance, using `npm rebuild` or `npm install`).', + '', + ].join('\n'), + ); + return 1; + } + + commandIo.stdout.write('What changed\n'); + commandIo.stdout.write(' Semantic layer comparison found 0 changes across 56 tables\n'); + commandIo.stdout.write(' New tables: 0\n'); + commandIo.stdout.write(' Changed tables: 0\n'); + commandIo.stdout.write(' Removed tables: 0\n'); + commandIo.stdout.write(' Unchanged tables: 56\n'); + return 0; + }); + const rebuildNativeSqlite = vi.fn(async () => 0); + + const result = await runKtxSetupDatabasesStep( + { + projectDir: tempDir, + inputMode: 'disabled', + databaseDrivers: ['postgres'], + databaseConnectionId: 'warehouse', + databaseUrl: 'env:DATABASE_URL', + databaseSchemas: [], + skipDatabases: false, + }, + io.io, + { + testConnection: vi.fn(async () => 0), + scanConnection, + rebuildNativeSqlite, + }, + ); + + expect(result.status).toBe('ready'); + expect(rebuildNativeSqlite).toHaveBeenCalledOnce(); + expect(rebuildNativeSqlite).toHaveBeenCalledWith(expect.anything()); + expect(scanConnection).toHaveBeenCalledTimes(2); + expect(io.stderr()).toContain('Native SQLite is built for a different Node.js ABI.'); + expect(io.stderr()).toContain('Rebuilding Native SQLite with pnpm run native:rebuild…'); + expect(io.stdout()).toContain('◇ Scan complete for warehouse'); + }); + it('writes Historic SQL config for supported Snowflake databases after validation succeeds', async () => { const io = makeIo(); const result = await runKtxSetupDatabasesStep( @@ -1325,7 +1378,6 @@ describe('setup databases step', () => { redactionPatterns: ['(?i)secret'], }, }); - expect(config.connections.snowflake.historicSql).not.toHaveProperty(legacyHistoricSqlServiceAccountPatternsKey); expect(config.ingest.adapters).toContain('historic-sql'); }); @@ -1373,10 +1425,8 @@ describe('setup databases step', () => { }, }, }); - expect(config.connections.warehouse.historicSql).not.toHaveProperty('minCalls'); expect(config.connections.warehouse.historicSql).not.toHaveProperty('windowDays'); expect(config.connections.warehouse.historicSql).not.toHaveProperty('redactionPatterns'); - expect(config.connections.warehouse.historicSql).not.toHaveProperty(legacyHistoricSqlServiceAccountPatternsKey); expect(config.ingest.adapters).toContain('historic-sql'); expect(config.ingest.workUnits.maxConcurrency).toBe(6); expect(io.stdout()).toContain('Historic SQL probe...'); @@ -1430,7 +1480,6 @@ describe('setup databases step', () => { redactionPatterns: [], }, }); - expect(config.connections.analytics.historicSql).not.toHaveProperty(legacyHistoricSqlServiceAccountPatternsKey); expect(config.ingest.adapters).toContain('historic-sql'); }); @@ -1480,7 +1529,6 @@ describe('setup databases step', () => { }, }, }); - expect(config.connections.warehouse.historicSql).not.toHaveProperty(legacyHistoricSqlServiceAccountPatternsKey); }); it('prints a non-blocking Postgres Historic SQL probe failure after connection test succeeds', async () => { diff --git a/packages/cli/src/setup-databases.ts b/packages/cli/src/setup-databases.ts index 58ee61d9..f697dd75 100644 --- a/packages/cli/src/setup-databases.ts +++ b/packages/cli/src/setup-databases.ts @@ -1,4 +1,8 @@ -import { writeFile } from 'node:fs/promises'; +import { execFile as execFileCallback } from 'node:child_process'; +import { readFile, writeFile } from 'node:fs/promises'; +import { delimiter, dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { promisify } from 'node:util'; import { cancel, confirm, isCancel, multiselect, password, select, text } from '@clack/prompts'; import type { HistoricSqlDialect } from '@ktx/context/ingest'; import { @@ -17,6 +21,7 @@ import { withSetupInterruptConfirmation } from './setup-interrupt.js'; import { writeProjectLocalSecretReference } from './setup-secrets.js'; const HISTORIC_SQL_WORK_UNIT_MAX_CONCURRENCY = 6; +const execFileAsync = promisify(execFileCallback); export type KtxSetupDatabaseDriver = | 'sqlite' @@ -39,7 +44,6 @@ export interface KtxSetupDatabasesArgs { disableHistoricSql?: boolean; historicSqlWindowDays?: number; historicSqlMinExecutions?: number; - historicSqlMinCalls?: number; historicSqlServiceAccountPatterns?: string[]; historicSqlRedactionPatterns?: string[]; skipDatabases: boolean; @@ -82,6 +86,7 @@ export interface KtxSetupDatabasesDeps { prompts?: KtxSetupDatabasesPromptAdapter; testConnection?: (projectDir: string, connectionId: string, io: KtxCliIo) => Promise; scanConnection?: (projectDir: string, connectionId: string, io: KtxCliIo) => Promise; + rebuildNativeSqlite?: (io: KtxCliIo) => Promise; listSchemas?: (projectDir: string, connectionId: string) => Promise; listTables?: (projectDir: string, connectionId: string) => Promise; historicSqlProbe?: KtxSetupHistoricSqlProbe; @@ -856,14 +861,13 @@ async function maybeApplyHistoricSqlConfig(input: { dialect, filters: historicSqlFiltersForSetup(input.args.historicSqlServiceAccountPatterns), }; - delete common[['serviceAccount', 'UserPatterns'].join('')]; if (dialect === 'postgres') { return { ...input.connection, historicSql: { ...common, - minExecutions: input.args.historicSqlMinExecutions ?? input.args.historicSqlMinCalls ?? 5, + minExecutions: input.args.historicSqlMinExecutions ?? 5, }, }; } @@ -959,6 +963,81 @@ function writePrefixedLines(write: (chunk: string) => void, output: string): voi } } +function envWithCurrentNodeFirst(env: NodeJS.ProcessEnv = process.env): NodeJS.ProcessEnv { + return { + ...env, + PATH: `${dirname(process.execPath)}${delimiter}${env.PATH ?? ''}`, + }; +} + +function errorTextProperty(error: unknown, property: 'stderr' | 'stdout'): string { + if (typeof error !== 'object' || error === null || !(property in error)) { + return ''; + } + const value = (error as Record)[property]; + return typeof value === 'string' ? value : ''; +} + +function commandFailureOutput(error: unknown): string { + const stderr = errorTextProperty(error, 'stderr'); + const stdout = errorTextProperty(error, 'stdout'); + const message = error instanceof Error ? error.message : String(error); + return [stderr.trim(), stdout.trim(), message.trim()].filter((line) => line.length > 0).join('\n'); +} + +type PackageJsonScriptStatus = 'has-script' | 'exists' | 'missing'; + +async function packageJsonScriptStatus( + packageJsonPath: string, + scriptName: string, +): Promise { + try { + const parsed = JSON.parse(await readFile(packageJsonPath, 'utf-8')) as unknown; + if (typeof parsed !== 'object' || parsed === null || !('scripts' in parsed)) { + return 'exists'; + } + const scripts = (parsed as { scripts?: unknown }).scripts; + return typeof scripts === 'object' && scripts !== null && scriptName in scripts ? 'has-script' : 'exists'; + } catch { + return 'missing'; + } +} + +async function nativeSqliteRebuildCommand(): Promise<{ cwd: string; args: string[] }> { + let dir = dirname(fileURLToPath(import.meta.url)); + let packageRoot: string | undefined; + while (true) { + const status = await packageJsonScriptStatus(join(dir, 'package.json'), 'native:rebuild'); + if (status === 'has-script') { + return { cwd: dir, args: ['run', 'native:rebuild'] }; + } + if (status === 'exists') { + packageRoot ??= dir; + } + + const parent = dirname(dir); + if (parent === dir) { + return { cwd: packageRoot ?? process.cwd(), args: ['rebuild', 'better-sqlite3'] }; + } + dir = parent; + } +} + +async function defaultRebuildNativeSqlite(io: KtxCliIo): Promise { + const command = await nativeSqliteRebuildCommand(); + try { + await execFileAsync('pnpm', command.args, { + cwd: command.cwd, + env: envWithCurrentNodeFirst(), + maxBuffer: 1024 * 1024 * 16, + }); + return 0; + } catch (error) { + writePrefixedLines((chunk) => io.stderr.write(chunk), commandFailureOutput(error)); + return typeof (error as { code?: unknown })?.code === 'number' ? (error as { code: number }).code : 1; + } +} + function flushPrefixedBufferedCommandOutput(io: KtxCliIo, bufferedIo: BufferedCommandIo): void { writePrefixedLines((chunk) => io.stdout.write(chunk), bufferedIo.stdoutText()); writePrefixedLines((chunk) => io.stderr.write(chunk), bufferedIo.stderrText()); @@ -1472,8 +1551,8 @@ async function validateAndScanConnection(input: { writeSetupSection(input.io, `Scanning ${input.connectionId}`, [ 'Running structural scan…', ]); - const scanIo = createBufferedCommandIo(); - const scanCode = await scanConnection(input.projectDir, input.connectionId, scanIo); + let scanIo = createBufferedCommandIo(); + let scanCode = await scanConnection(input.projectDir, input.connectionId, scanIo); if (scanCode !== 0) { const nativeSqliteDetail = nativeSqliteAbiMismatchDetail(`${scanIo.stderrText()}\n${scanIo.stdoutText()}`); if (nativeSqliteDetail) { @@ -1483,10 +1562,32 @@ async function validateAndScanConnection(input: { `Structural scan failed for ${input.connectionId}.`, 'Native SQLite is built for a different Node.js ABI.', `Detail: ${nativeSqliteDetail}`, - 'Fix: pnpm run native:rebuild', - `Retry: ktx scan --project-dir ${input.projectDir} ${input.connectionId}`, + 'Rebuilding Native SQLite with pnpm run native:rebuild…', ].join('\n'), ); + const rebuildNativeSqlite = input.deps.rebuildNativeSqlite ?? defaultRebuildNativeSqlite; + const rebuildCode = await rebuildNativeSqlite(input.io); + if (rebuildCode === 0) { + writePrefixedLines( + (chunk) => input.io.stderr.write(chunk), + 'Native SQLite rebuild complete. Retrying structural scan…', + ); + const retryScanIo = createBufferedCommandIo(); + scanCode = await scanConnection(input.projectDir, input.connectionId, retryScanIo); + scanIo = retryScanIo; + } + if (scanCode !== 0) { + writePrefixedLines( + (chunk) => input.io.stderr.write(chunk), + [ + rebuildCode === 0 + ? `Structural scan still failed for ${input.connectionId} after rebuilding Native SQLite.` + : `Native SQLite rebuild failed for ${input.connectionId}.`, + 'Fix: pnpm run native:rebuild', + `Retry: ktx scan --project-dir ${input.projectDir} ${input.connectionId}`, + ].join('\n'), + ); + } } else { flushPrefixedBufferedCommandOutput(input.io, scanIo); writePrefixedLines( @@ -1497,7 +1598,9 @@ async function validateAndScanConnection(input: { ].join('\n'), ); } - return false; + if (scanCode !== 0) { + return false; + } } const scanOutput = scanIo.stdoutText(); const reportPath = readOutputValue(scanOutput, 'Report'); diff --git a/packages/cli/src/setup-sources.ts b/packages/cli/src/setup-sources.ts index 313dfbe0..6ab71106 100644 --- a/packages/cli/src/setup-sources.ts +++ b/packages/cli/src/setup-sources.ts @@ -544,8 +544,8 @@ function sourcePathFromFileRepoUrl(repoUrl: string, subpath?: string): string { } function repoAuthToken(connection: KtxProjectConnectionConfig | Record): string | null { - const ref = stringField(connection.auth_token_ref) ?? stringField(connection.authTokenRef); - const literal = stringField(connection.authToken) ?? stringField(connection.auth_token); + const ref = stringField(connection.auth_token_ref); + const literal = stringField(connection.auth_token); return literal ?? resolveKtxConfigReference(ref, process.env) ?? null; } @@ -563,8 +563,8 @@ async function collectYamlFilesRecursive(sourceRoot: string): Promise { - let sourceDir = stringField(connection.source_dir) ?? stringField(connection.sourceDir); - const repoUrl = stringField(connection.repo_url) ?? stringField(connection.repoUrl); + let sourceDir = stringField(connection.source_dir); + const repoUrl = stringField(connection.repo_url); if (!sourceDir && repoUrl?.startsWith('file:')) { sourceDir = sourcePathFromFileRepoUrl(repoUrl, stringField(connection.path)); } @@ -624,7 +624,7 @@ async function defaultValidateLooker(projectDir: string, connectionId: string): } async function defaultValidateLookml(connection: KtxProjectConnectionConfig): Promise { - const repoUrl = stringField(connection.repoUrl) ?? stringField(connection.repo_url); + const repoUrl = stringField(connection.repoUrl); if (!repoUrl) { return { ok: false, message: 'LookML setup requires repoUrl.' }; } diff --git a/packages/cli/src/setup.ts b/packages/cli/src/setup.ts index 1ab48f0b..0dc0d7cd 100644 --- a/packages/cli/src/setup.ts +++ b/packages/cli/src/setup.ts @@ -90,7 +90,6 @@ export type KtxSetupArgs = disableHistoricSql?: boolean; historicSqlWindowDays?: number; historicSqlMinExecutions?: number; - historicSqlMinCalls?: number; historicSqlServiceAccountPatterns?: string[]; historicSqlRedactionPatterns?: string[]; skipDatabases: boolean; @@ -636,7 +635,6 @@ async function runKtxSetupInner(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetup ...(args.historicSqlMinExecutions !== undefined ? { historicSqlMinExecutions: args.historicSqlMinExecutions } : {}), - ...(args.historicSqlMinCalls !== undefined ? { historicSqlMinCalls: args.historicSqlMinCalls } : {}), ...(args.historicSqlServiceAccountPatterns ? { historicSqlServiceAccountPatterns: args.historicSqlServiceAccountPatterns } : {}), diff --git a/packages/cli/src/sl.test.ts b/packages/cli/src/sl.test.ts index 48c7f4c7..ff4132b4 100644 --- a/packages/cli/src/sl.test.ts +++ b/packages/cli/src/sl.test.ts @@ -129,7 +129,7 @@ describe('runKtxSl', () => { connectionId: 'warehouse', name: 'orders', score: expect.any(Number), - matchReasons: expect.arrayContaining(['token']), + matchReasons: expect.any(Array), }), ], }, diff --git a/packages/context/prompts/memory_agent_bundle_ingest_reconcile.md b/packages/context/prompts/memory_agent_bundle_ingest_reconcile.md index 5d2316fd..30b52537 100644 --- a/packages/context/prompts/memory_agent_bundle_ingest_reconcile.md +++ b/packages/context/prompts/memory_agent_bundle_ingest_reconcile.md @@ -1,5 +1,5 @@ -You are the reconciliation agent for a multi-file ingest bundle. Stage 3 WorkUnits have already run against this job's session worktree; your input is the deterministic Stage Index listing every write each WU made, plus an Eviction Set listing raw files present in the prior sync but absent in this one. Your job is to (a) decide what happens to each evicted artifact (remove vs retain with a deprecation marker), (b) sweep the Stage Index for any cross-WU conflicts the individual WUs missed, and (c) emit conflict + eviction records that the runner will fold into the final IngestReport. +You are the reconciliation agent for a multi-file ingest bundle. Stage 3 WorkUnits have already run against this job's session worktree; your input is the deterministic Stage Index listing every write each WU made, plus an Eviction Set listing raw files present in the prior sync but absent in this one. Your job is to (a) remove artifacts produced by deleted raw files, (b) sweep the Stage Index for any cross-WU conflicts the individual WUs missed, and (c) emit conflict + eviction records that the runner will fold into the final IngestReport. @@ -12,7 +12,7 @@ Parsimonious. Stage 3 WUs already loaded `ingest_triage` and handled conflicts t 3. If the system prompt includes ``, apply those pins before flagging a same-name or near-duplicate conflict. A pinned `canonicalArtifactKey` keeps the contested name when it is present in the Stage Index; competing variants keep or receive disambiguated names. 4. Sweep both exact-key conflicts and near-duplicate writes. Compare WUs that wrote overlapping SL source names, overlapping wiki keys, the same `tables:` or `sl_refs:` action details, or obviously equivalent topic titles under different wiki keys. Call `stage_diff` to see the actual difference, and use `wiki_read`/`sl_read_source` when two different keys appear to describe the same table, metric, or source-of-truth mapping. If they're the same content, leave one canonical artifact and record the duplicate as subsumed. If they differ per `ingest_triage` rules, apply the correct resolution (rename + capture; election of canonical; silent replace for expression-only re-ingest change; or pinned canonical), then call `emit_conflict_resolution` with the artifact key and decision. 5. For any `wiki_write`, `wiki_remove`, `sl_write_source`, or `sl_edit_source` call you make during reconciliation, include `rawPaths` with only the raw paths that directly caused that reconciliation action. -6. Call `eviction_list()` for deleted raw paths. For each eviction: if inbound refs are empty, remove the artifact (`sl_delete`, `wiki_remove`) and include that evicted raw path in `rawPaths`; if inbound refs exist, retain with a deprecation marker and include that evicted raw path in `rawPaths`. Then call `emit_eviction_decision` for every removed or retained artifact. +6. Call `eviction_list()` for deleted raw paths. For each listed artifact, remove it (`sl_delete`, `wiki_remove`) and include the evicted raw path in `rawPaths`. Then call `emit_eviction_decision` with `action: "removed"` for every removed artifact. 7. If the Stage 4 sweep discovers a raw file whose only honest outcome is standalone SQL, wiki-only capture, or a human flag, call `emit_unmapped_fallback` with the raw path, reason, and fallback kind. 8. Use `read_raw_span` to zoom into specific raw files when you need to resolve what two contested measures or wiki pages actually describe. 9. Exit when you've processed every item. diff --git a/packages/context/skills/ingest_triage/SKILL.md b/packages/context/skills/ingest_triage/SKILL.md index 1ac3d108..df13ed83 100644 --- a/packages/context/skills/ingest_triage/SKILL.md +++ b/packages/context/skills/ingest_triage/SKILL.md @@ -32,8 +32,8 @@ Apply the rules below before every write that could collide with an existing art | Definitional contradiction | Same name, substantively different formulas (different aggregation, different filters, different columns) | **Rename + capture**: disambiguate ALL variants with suffix derived from the domain (`churn_risk_engagement_based`, `churn_risk_billing_based`) and write a unified wiki page listing every variant with provenance. The contested name does NOT land in the SL. **Always flag.** | 5. **Eviction (Stage 4 only)**: for each entry in `eviction_list()`: - - `inbound_refs: []` → remove the artifact (`sl_delete` for SL sources, `wiki_remove` for wiki pages). - - `inbound_refs: [...]` → retain the artifact, set `deprecated: true` on SL sources (via `sl_edit_source`), write a wiki note "origin file removed in ; preserved because referenced by: …". Flag in the IngestReport so the user can plan migration. + - Remove the artifact (`sl_delete` for SL sources, `wiki_remove` for wiki pages). + - Record the removal with `emit_eviction_decision` and `action: "removed"`. ## Why same-ingest vs re-ingest differs diff --git a/packages/context/skills/metabase_ingest/SKILL.md b/packages/context/skills/metabase_ingest/SKILL.md index f5aa00e2..d35166dc 100644 --- a/packages/context/skills/metabase_ingest/SKILL.md +++ b/packages/context/skills/metabase_ingest/SKILL.md @@ -98,7 +98,7 @@ measures: expr: "" ``` -Overlay shape: `name:` plus any of `measures:`, `segments:`, `description:`, `joins:`, `disable_joins:`. Never include `sql:`, `table:`, `grain:`, or `columns:` on a manifest-backed name — those would shadow the manifest's schema and drop its joins. Overlay `joins:` are merged additively with the manifest's joins (deduped by `to` + `on`); use `disable_joins: [""]` to suppress a specific manifest join. After the overlay exists, use `sl_edit_source` for further tweaks. See `sl_capture` skill for the canonical overlay rule. +Overlay shape: `name:` plus any of `measures:`, `segments:`, `descriptions:`, `joins:`, `disable_joins:`. Never include `sql:`, `table:`, `grain:`, or `columns:` on a manifest-backed name — those would shadow the manifest's schema and drop its joins. Overlay `joins:` are merged additively with the manifest's joins (deduped by `to` + `on`); use `disable_joins: [""]` to suppress a specific manifest join. After the overlay exists, use `sl_edit_source` for further tweaks. See `sl_capture` skill for the canonical overlay rule. **Join discovery:** When your card's SQL references warehouse tables (e.g. in `FROM` or `JOIN` clauses), call `sl_discover({ query: '' })` before writing. The matching manifest entry's `name` is the value you use in `joins: [- to: ]` only when the card output exposes a local key that matches the target source grain (for example `account_id = mart_account_segments.account_id`). Do not declare a KTX join just because the card SQL joins that table internally. If the output only exposes display fields such as `account_name`, keep the SQL source self-contained or project the key before adding the join. Use `many_to_one` for FK-to-dimension joins, `one_to_many` for the reverse. diff --git a/packages/context/skills/metricflow_ingest/SKILL.md b/packages/context/skills/metricflow_ingest/SKILL.md index 47187ffb..6ed4b916 100644 --- a/packages/context/skills/metricflow_ingest/SKILL.md +++ b/packages/context/skills/metricflow_ingest/SKILL.md @@ -177,7 +177,8 @@ semantic_models: # KTX overlay at /orders.yaml: # name: orders -description: Order fact table. +descriptions: + user: Order fact table. measures: - {name: order_count, expr: "count(order_id)"} - {name: gross_amount, expr: "sum(amount)"} @@ -221,7 +222,8 @@ metrics: # # name: orders_ext -description: Extended order fact including refund handling; `revenue` = gross - refund. +descriptions: + user: Extended order fact including refund handling; `revenue` = gross - refund. measures: - {name: order_count, expr: "count(order_id)"} - {name: gross_amount, expr: "sum(amount)"} diff --git a/packages/context/skills/sl/SKILL.md b/packages/context/skills/sl/SKILL.md index 9cdb8b34..f7077c33 100644 --- a/packages/context/skills/sl/SKILL.md +++ b/packages/context/skills/sl/SKILL.md @@ -29,7 +29,8 @@ Enrich a manifest-backed table with measures, computed columns, joins, and segme ```yaml name: fct_orders # must match an existing manifest table -description: "Overlay adding business measures to the orders fact table." +descriptions: + user: "Overlay adding business measures to the orders fact table." measures: - name: total_revenue expr: sum(amount) diff --git a/packages/context/skills/sl_capture/SKILL.md b/packages/context/skills/sl_capture/SKILL.md index 4bc383eb..a40111ea 100644 --- a/packages/context/skills/sl_capture/SKILL.md +++ b/packages/context/skills/sl_capture/SKILL.md @@ -100,13 +100,13 @@ measures: **Extract repeated filter bundles into named segments.** If the same predicate appears on multiple measures of the same source, lift it to a `segments[]` entry and have each measure reference it. One edit updates every measure that depends on it. -**Never write a standalone file on a manifest-backed name.** If `sl_discover({ tableName })` finds an existing schema for that name, you MUST write an overlay (`name:` + `measures:`/`segments:`/`description:` only — no `sql:`, `table:`, `grain:`, `columns:`, `joins:`). A standalone with `sql:` or `table:` on a manifest-backed name clobbers the inherited columns and joins; `sl_write_source` and `sl_validate` both reject this shape with a clear fix hint. Always run `sl_discover` before your first write on any existing name. +**Never write a standalone file on a manifest-backed name.** If `sl_discover({ tableName })` finds an existing schema for that name, you MUST write an overlay (`name:` + `measures:`/`segments:`/`descriptions:` only — no `sql:`, `table:`, `grain:`, `columns:`, `joins:`). A standalone with `sql:` or `table:` on a manifest-backed name clobbers the inherited columns and joins; `sl_write_source` and `sl_validate` both reject this shape with a clear fix hint. Always run `sl_discover` before your first write on any existing name. **Prefer overlay decomposition over standalone SQL sources.** Before reaching for `source_type: sql`, check whether the metric decomposes into measures on existing overlays (including cross-source derived measures). Use `source_type: sql` only when: - The metric requires per-user/per-entity derivation that cannot be expressed as a single `expr` (e.g., `EXISTS` over a time-windowed subset), OR - The metric requires multi-step CTEs whose intermediate grain is not a column in any existing source. -When an `sql` source is unavoidable, note in its `description` which SL gap forced the choice so it can be retired once the primitive ships. It must target a name NOT in the manifest — pick a distinct one (e.g. `mrr_waterfall_rollup`, not `fct_orders`). +When an `sql` source is unavoidable, note in its `descriptions` map which SL gap forced the choice so it can be retired once the primitive ships. It must target a name NOT in the manifest — pick a distinct one (e.g. `mrr_waterfall_rollup`, not `fct_orders`). ## Slim standalone sources via `inherits_columns_from` @@ -116,7 +116,8 @@ Discover the manifest key with `sl_discover` — pass the bare name (`CONSIGNMEN ```yaml name: aav_consignments -description: AAV consignments — filtered view of MARTS.CONSIGNMENTS for the auto-auction-vaulting channel. +descriptions: + user: AAV consignments — filtered view of MARTS.CONSIGNMENTS for the auto-auction-vaulting channel. source_type: sql sql: | SELECT CONSIGNED_ITEM_ID, CASH_ADV_AMOUNT, ALT_VALUE_COMBINED, my_derived_flag @@ -127,10 +128,10 @@ sql: | inherits_columns_from: CONSIGNMENTS grain: [CONSIGNED_ITEM_ID] columns: - - { name: CONSIGNED_ITEM_ID } # type/description inherited from manifest + - { name: CONSIGNED_ITEM_ID } # type/descriptions inherited from manifest - { name: CASH_ADV_AMOUNT } - { name: ALT_VALUE_COMBINED } - - { name: my_derived_flag, type: boolean, expr: "CASH_ADV_AMOUNT > 0", description: "Computed locally — has any cash advance." } + - { name: my_derived_flag, type: boolean, expr: "CASH_ADV_AMOUNT > 0", descriptions: { user: "Computed locally — has any cash advance." } } measures: - name: total_cash_advance expr: sum(CASH_ADV_AMOUNT) diff --git a/packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts b/packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts index c2c679e5..36461bb2 100644 --- a/packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts +++ b/packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.test.ts @@ -13,7 +13,7 @@ async function tempDir(): Promise { const sqlAnalysis: SqlAnalysisPort = { async analyzeForFingerprint() { - throw new Error('legacy analyzeForFingerprint must not be used'); + throw new Error('analyzeForFingerprint must not be used'); }, async analyzeBatch() { return new Map(); @@ -66,7 +66,7 @@ describe('HistoricSqlSourceAdapter', () => { }; const batchSqlAnalysis: SqlAnalysisPort = { async analyzeForFingerprint() { - throw new Error('legacy analyzeForFingerprint must not be used'); + throw new Error('analyzeForFingerprint must not be used'); }, async analyzeBatch() { return new Map([ diff --git a/packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.ts b/packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.ts index aee051e7..be2fc9f0 100644 --- a/packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.ts +++ b/packages/context/src/ingest/adapters/historic-sql/historic-sql.adapter.ts @@ -1,5 +1,3 @@ -import { rm } from 'node:fs/promises'; -import { join } from 'node:path'; import type { ChunkResult, DiffSet, FetchContext, ScopeDescriptor, SourceAdapter } from '../../types.js'; import { chunkHistoricSqlUnifiedStagedDir, describeHistoricSqlUnifiedScope } from './chunk-unified.js'; import { detectHistoricSqlStagedDir } from './detect.js'; @@ -28,11 +26,6 @@ export class HistoricSqlSourceAdapter implements SourceAdapter { pullConfig, now: this.deps.now?.(), }); - if (this.deps.legacyPostgresBaselineRootDir) { - await rm(join(this.deps.legacyPostgresBaselineRootDir, ctx.connectionId, ['pgss', 'baseline.json'].join('-')), { - force: true, - }); - } } chunk(stagedDir: string, diffSet?: DiffSet): Promise { diff --git a/packages/context/src/ingest/adapters/historic-sql/projection.test.ts b/packages/context/src/ingest/adapters/historic-sql/projection.test.ts index f2a5b068..95adf13f 100644 --- a/packages/context/src/ingest/adapters/historic-sql/projection.test.ts +++ b/packages/context/src/ingest/adapters/historic-sql/projection.test.ts @@ -284,7 +284,7 @@ describe('projectHistoricSqlEvidence', () => { ); }); - it('marks missing table usage stale and deletes legacy historic SQL query pages', async () => { + it('marks missing table usage stale without deleting old query pages', async () => { const workdir = await tempWorkdir(); await writeText( workdir, @@ -322,22 +322,22 @@ describe('projectHistoricSqlEvidence', () => { }); await writeText( workdir, - 'knowledge/global/historic-sql-legacy-template.md', + 'knowledge/global/historic-sql-old-template.md', [ '---', YAML.stringify({ - summary: 'Legacy template page', + summary: 'Old template page', tags: ['historic-sql', 'query-pattern'], refs: [], sl_refs: ['orders'], usage_mode: 'auto', source: 'historic-sql', tables: ['public.orders'], - fingerprints: ['legacy:1'], + fingerprints: ['old:1'], }).trimEnd(), '---', '', - 'Legacy body', + 'Old body', '', ].join('\n'), ); @@ -345,7 +345,6 @@ describe('projectHistoricSqlEvidence', () => { const result = await projectHistoricSqlEvidence({ workdir, connectionId: 'warehouse', syncId: 'sync-1', runId: 'run-1' }); expect(result.staleTablesMarked).toBe(1); - expect(result.legacyPagesDeleted).toBe(1); expect(result.touchedSources).toEqual([{ connectionId: 'warehouse', sourceName: 'orders' }]); const shard = YAML.parse(await readFile(join(workdir, 'semantic-layer/warehouse/_schema/public.yaml'), 'utf-8')); expect(shard.tables.orders.usage).toEqual({ @@ -357,8 +356,8 @@ describe('projectHistoricSqlEvidence', () => { commonJoins: [], staleSince: '2026-05-11T00:00:00.000Z', }); - await expect(readFile(join(workdir, 'knowledge/global/historic-sql-legacy-template.md'), 'utf-8')).rejects.toMatchObject({ - code: 'ENOENT', - }); + await expect(readFile(join(workdir, 'knowledge/global/historic-sql-old-template.md'), 'utf-8')).resolves.toContain( + 'Old body', + ); }); }); diff --git a/packages/context/src/ingest/adapters/historic-sql/projection.ts b/packages/context/src/ingest/adapters/historic-sql/projection.ts index 25a317f3..7d4da94f 100644 --- a/packages/context/src/ingest/adapters/historic-sql/projection.ts +++ b/packages/context/src/ingest/adapters/historic-sql/projection.ts @@ -1,4 +1,4 @@ -import { access, mkdir, readdir, readFile, rename, rm, writeFile } from 'node:fs/promises'; +import { access, mkdir, readdir, readFile, rename, writeFile } from 'node:fs/promises'; import { dirname, join, relative } from 'node:path'; import YAML from 'yaml'; import { rawSourcesDirForSync } from '../../raw-sources-paths.js'; @@ -20,7 +20,6 @@ export interface HistoricSqlProjectionResult { patternPagesWritten: number; stalePatternPagesMarked: number; archivedPatternPages: number; - legacyPagesDeleted: number; touchedSources: Array<{ connectionId: string; sourceName: string }>; warnings: string[]; } @@ -152,11 +151,6 @@ function isHistoricPatternPage(page: HistoricSqlPatternPage): boolean { ); } -function isLegacyQueryPage(page: HistoricSqlPatternPage): boolean { - const tags = Array.isArray(page.frontmatter.tags) ? page.frontmatter.tags : []; - return page.frontmatter.source === 'historic-sql' && tags.includes('query-pattern') && !tags.includes('pattern'); -} - function isArchivedPatternPage(page: HistoricSqlPatternPage): boolean { const tags = Array.isArray(page.frontmatter.tags) ? page.frontmatter.tags : []; return tags.includes('archived'); @@ -228,7 +222,6 @@ export async function projectHistoricSqlEvidence(input: HistoricSqlProjectionInp patternPagesWritten: 0, stalePatternPagesMarked: 0, archivedPatternPages: 0, - legacyPagesDeleted: 0, touchedSources: [], warnings: [], }; @@ -333,10 +326,5 @@ export async function projectHistoricSqlEvidence(input: HistoricSqlProjectionInp result.stalePatternPagesMarked += 1; } - for (const page of allPages.filter(isLegacyQueryPage)) { - await rm(page.path, { force: true }); - result.legacyPagesDeleted += 1; - } - return result; } diff --git a/packages/context/src/ingest/adapters/historic-sql/types.test.ts b/packages/context/src/ingest/adapters/historic-sql/types.test.ts index 076e5d8e..f5a6f853 100644 --- a/packages/context/src/ingest/adapters/historic-sql/types.test.ts +++ b/packages/context/src/ingest/adapters/historic-sql/types.test.ts @@ -8,7 +8,7 @@ import { } from './types.js'; describe('historic-sql unified contracts', () => { - it('parses minExecutions and accepts minCalls as a one-release alias', () => { + it('parses minExecutions and service-account filters', () => { expect(historicSqlUnifiedPullConfigSchema.parse({ dialect: 'postgres', minExecutions: 9 })).toMatchObject({ dialect: 'postgres', minExecutions: 9, @@ -18,7 +18,15 @@ describe('historic-sql unified contracts', () => { staleArchiveAfterDays: 90, }); - expect(historicSqlUnifiedPullConfigSchema.parse({ dialect: 'postgres', minCalls: 7 }).minExecutions).toBe(7); + const parsed = historicSqlUnifiedPullConfigSchema.parse({ + dialect: 'postgres', + minExecutions: 7, + filters: { + serviceAccounts: { patterns: ['^svc_'], mode: 'exclude' }, + }, + }); + expect(parsed.minExecutions).toBe(7); + expect(parsed.filters.serviceAccounts).toEqual({ patterns: ['^svc_'], mode: 'exclude' }); }); it('validates aggregate templates from warehouse readers', () => { diff --git a/packages/context/src/ingest/adapters/historic-sql/types.ts b/packages/context/src/ingest/adapters/historic-sql/types.ts index a827e8ae..07711d52 100644 --- a/packages/context/src/ingest/adapters/historic-sql/types.ts +++ b/packages/context/src/ingest/adapters/historic-sql/types.ts @@ -8,26 +8,7 @@ export type HistoricSqlDialect = z.infer; const filterModeSchema = z.enum(['exclude', 'include', 'mark-only']); -function isRecord(value: unknown): value is Record { - return typeof value === 'object' && value !== null && !Array.isArray(value); -} - -export const historicSqlUnifiedPullConfigSchema = z.preprocess((value) => { - if (!isRecord(value)) { - return value; - } - const next: Record = { ...value }; - if (next.minExecutions === undefined && typeof next.minCalls === 'number') { - next.minExecutions = next.minCalls; - } - if (!next.filters && Array.isArray(next.serviceAccountUserPatterns)) { - next.filters = { - serviceAccounts: { patterns: next.serviceAccountUserPatterns, mode: 'exclude' }, - dropTrivialProbes: true, - }; - } - return next; -}, z.object({ +export const historicSqlUnifiedPullConfigSchema = z.object({ dialect: historicSqlDialectSchema, windowDays: z.number().int().positive().default(90), minExecutions: z.number().int().nonnegative().default(5), @@ -48,7 +29,7 @@ export const historicSqlUnifiedPullConfigSchema = z.preprocess((value) => { }).default({ dropTrivialProbes: true }), redactionPatterns: z.array(z.string()).default([]), staleArchiveAfterDays: z.number().int().positive().default(90), -})); +}); export type HistoricSqlUnifiedPullConfig = z.infer; @@ -157,6 +138,5 @@ export interface HistoricSqlSourceAdapterDeps { sqlAnalysis: SqlAnalysisPort; reader: HistoricSqlReader; queryClient: unknown; - legacyPostgresBaselineRootDir?: string; now?: () => Date; } diff --git a/packages/context/src/ingest/adapters/looker/local-looker.adapter.ts b/packages/context/src/ingest/adapters/looker/local-looker.adapter.ts index a29fecd1..47299373 100644 --- a/packages/context/src/ingest/adapters/looker/local-looker.adapter.ts +++ b/packages/context/src/ingest/adapters/looker/local-looker.adapter.ts @@ -26,13 +26,11 @@ export function lookerCredentialsFromLocalConnection( if (!connection || String(connection.driver).toLowerCase() !== 'looker') { throw new Error(`Connection "${connectionId}" is not a Looker connection`); } - const baseUrl = stringField(connection.base_url) ?? stringField(connection.baseUrl) ?? stringField(connection.url); - const clientId = stringField(connection.client_id) ?? stringField(connection.clientId); + const baseUrl = stringField(connection.base_url); + const clientId = stringField(connection.client_id); const clientSecret = stringField(connection.client_secret) ?? - stringField(connection.clientSecret) ?? - (stringField(connection.client_secret_ref) ? resolveEnvReference(String(connection.client_secret_ref), env) : null) ?? - (stringField(connection.clientSecretRef) ? resolveEnvReference(String(connection.clientSecretRef), env) : null); + (stringField(connection.client_secret_ref) ? resolveEnvReference(String(connection.client_secret_ref), env) : null); if (!baseUrl) { throw new Error(`Connection "${connectionId}" is missing Looker base_url`); diff --git a/packages/context/src/ingest/adapters/metabase/client-port.test.ts b/packages/context/src/ingest/adapters/metabase/client-port.test.ts index 9686e552..8f775b56 100644 --- a/packages/context/src/ingest/adapters/metabase/client-port.test.ts +++ b/packages/context/src/ingest/adapters/metabase/client-port.test.ts @@ -87,10 +87,13 @@ it('allows the concrete client result shapes used by the relocated Metabase clie const datasetQuery: MetabaseDatasetQuery = { type: 'native', database: 42, - native: { - query: 'SELECT * FROM orders WHERE created_at > {{ created_at }}', - 'template-tags': { created_at: templateTag }, - }, + stages: [ + { + 'lib/type': 'mbql.stage/native', + native: 'SELECT * FROM orders WHERE created_at > {{ created_at }}', + 'template-tags': { created_at: templateTag }, + }, + ], }; const card: MetabaseCard = { id: 1, diff --git a/packages/context/src/ingest/adapters/metabase/client-port.ts b/packages/context/src/ingest/adapters/metabase/client-port.ts index 7aa1f3ed..a5fdb6ce 100644 --- a/packages/context/src/ingest/adapters/metabase/client-port.ts +++ b/packages/context/src/ingest/adapters/metabase/client-port.ts @@ -117,7 +117,7 @@ interface MetabaseNativeStage { } interface MetabaseLegacyNativeQuery { - query: string; + query?: string; 'template-tags'?: Record; } diff --git a/packages/context/src/ingest/adapters/metabase/client.test.ts b/packages/context/src/ingest/adapters/metabase/client.test.ts index 1ee3fe93..1c0fdfa9 100644 --- a/packages/context/src/ingest/adapters/metabase/client.test.ts +++ b/packages/context/src/ingest/adapters/metabase/client.test.ts @@ -32,10 +32,22 @@ function nativeCard(query: string, templateTags: Record = {}): MetabaseCard { + return { + id: 1, + name: 'Legacy native card', + type: 'model', + query_type: 'native', + database_id: 6, + dataset_query: { + type: 'native', + database: 6, + native: { query, 'template-tags': templateTags }, }, }; } @@ -277,6 +289,25 @@ describe('getDummyValueForWidgetType', () => { }); }); +describe('MetabaseClient legacy native dataset query support', () => { + it('reads SQL and template tags from dataset_query.native', async () => { + const client = new MetabaseClient(runtime, fastRetryConfig); + const card = legacyNativeCard('SELECT * FROM orders WHERE status = {{ status }}', { + status: { + name: 'status', + type: 'text', + default: 'paid', + }, + }); + + expect(client.getNativeSql(card)).toBe('SELECT * FROM orders WHERE status = {{ status }}'); + expect(client.getTemplateTags(card)).toEqual({ + status: expect.objectContaining({ name: 'status', type: 'text' }), + }); + await expect(client.getCardSql(card)).resolves.toBe('SELECT * FROM orders WHERE status = {{ status }}'); + }); +}); + describe('MetabaseClient.getResolvedSql', () => { function makeClient(setup?: (client: MetabaseClient) => void): MetabaseClient { const client = new MetabaseClient({ apiUrl: 'http://test', apiKey: 'k' }); @@ -318,7 +349,7 @@ describe('MetabaseClient.getResolvedSql', () => { dataset_query: { type: 'native', database: 6, - native: { query: 'SELECT a, b FROM base' }, + stages: [{ 'lib/type': 'mbql.stage/native', native: 'SELECT a, b FROM base' }], }, }); const client = makeClient((client) => { diff --git a/packages/context/src/ingest/adapters/metabase/client.ts b/packages/context/src/ingest/adapters/metabase/client.ts index 2b70bc79..1962bfe0 100644 --- a/packages/context/src/ingest/adapters/metabase/client.ts +++ b/packages/context/src/ingest/adapters/metabase/client.ts @@ -150,7 +150,7 @@ function injectNativeSql(datasetQuery: MetabaseDatasetQuery, sql: string): Metab stages[0] = { ...stages[0], native: sql }; return { ...datasetQuery, stages }; } - if (datasetQuery?.native) { + if (datasetQuery?.native?.query !== undefined) { return { ...datasetQuery, native: { ...datasetQuery.native, query: sql } }; } return datasetQuery; @@ -370,36 +370,12 @@ export class MetabaseClient implements MetabaseRuntimeClient { }); } - /** - * Extract native SQL from card, handling both pMBQL (v57+) and legacy formats. - * - pMBQL format: dataset_query.stages[0].native - * - Legacy format: dataset_query.native.query - */ getNativeSql(card: MetabaseCard): string | null { - // pMBQL format (v57+): stages[0].native - const pMbqlSql = card.dataset_query?.stages?.[0]?.native; - if (pMbqlSql) { - return pMbqlSql; - } - - // Legacy format: native.query - return card.dataset_query?.native?.query ?? null; + return card.dataset_query?.stages?.[0]?.native ?? card.dataset_query?.native?.query ?? null; } - /** - * Extract template tags from card, handling both pMBQL and legacy formats. - * - pMBQL format: dataset_query.stages[0]['template-tags'] - * - Legacy format: dataset_query.native['template-tags'] - */ getTemplateTags(card: MetabaseCard): Record { - // pMBQL format: stages[0]['template-tags'] - const pMbqlTags = card.dataset_query?.stages?.[0]?.['template-tags']; - if (pMbqlTags) { - return pMbqlTags; - } - - // Legacy format: native['template-tags'] - return card.dataset_query?.native?.['template-tags'] ?? {}; + return card.dataset_query?.stages?.[0]?.['template-tags'] ?? card.dataset_query?.native?.['template-tags'] ?? {}; } async getCardSql(card: MetabaseCard): Promise { diff --git a/packages/context/src/ingest/adapters/metabase/local-metabase.adapter.test.ts b/packages/context/src/ingest/adapters/metabase/local-metabase.adapter.test.ts index 0c854f6d..7cbe913b 100644 --- a/packages/context/src/ingest/adapters/metabase/local-metabase.adapter.test.ts +++ b/packages/context/src/ingest/adapters/metabase/local-metabase.adapter.test.ts @@ -48,19 +48,6 @@ describe('metabaseRuntimeConfigFromLocalConnection', () => { }); }); - it('accepts url as the local api URL alias', () => { - const connection: KtxProjectConnectionConfig = { - driver: 'metabase', - url: 'https://metabase.example.com', - api_key: 'literal-test-key', // pragma: allowlist secret - }; - - expect(metabaseRuntimeConfigFromLocalConnection('prod-metabase', connection)).toEqual({ - apiUrl: 'https://metabase.example.com', - apiKey: 'literal-test-key', // pragma: allowlist secret - }); - }); - it('rejects proxy-bearing local Metabase connections', () => { const connection: KtxProjectConnectionConfig = { driver: 'metabase', diff --git a/packages/context/src/ingest/adapters/metabase/local-metabase.adapter.ts b/packages/context/src/ingest/adapters/metabase/local-metabase.adapter.ts index a7ffc5de..8d8d5f06 100644 --- a/packages/context/src/ingest/adapters/metabase/local-metabase.adapter.ts +++ b/packages/context/src/ingest/adapters/metabase/local-metabase.adapter.ts @@ -37,9 +37,9 @@ export function metabaseRuntimeConfigFromLocalConnection( ); } - const apiUrl = stringField(connection.api_url) ?? stringField(connection.apiUrl) ?? stringField(connection.url); - const literalApiKey = stringField(connection.api_key) ?? stringField(connection.apiKey); - const apiKeyRef = stringField(connection.api_key_ref) ?? stringField(connection.apiKeyRef); + const apiUrl = stringField(connection.api_url); + const literalApiKey = stringField(connection.api_key); + const apiKeyRef = stringField(connection.api_key_ref); const apiKey = literalApiKey ?? (apiKeyRef ? resolveKtxConfigReference(apiKeyRef, env) : null); if (!apiUrl) { diff --git a/packages/context/src/ingest/adapters/metricflow/import-semantic-models.ts b/packages/context/src/ingest/adapters/metricflow/import-semantic-models.ts index bfdd824f..13127a3d 100644 --- a/packages/context/src/ingest/adapters/metricflow/import-semantic-models.ts +++ b/packages/context/src/ingest/adapters/metricflow/import-semantic-models.ts @@ -14,7 +14,6 @@ import { getMetricflowAvailableColumnNames, mapCrossModelMetricToSource, resolveMetricflowSemanticModelSourceName, - toKebabCaseMetricflowName, type MetricflowHostTable, type MetricflowSemanticModelImportContext, } from './semantic-models.js'; @@ -129,16 +128,6 @@ export async function importMetricflowSemanticModels( { skipValidation: true }, ); - const legacyWarning = await legacyKebabSourceWarning( - semanticLayerService, - input.connectionId, - context.model.modelRef, - context.sourceName, - ); - if (legacyWarning) { - warnings.push(legacyWarning); - } - if (existing) { sourcesUpdated++; } else { @@ -234,26 +223,6 @@ async function resolveManifestSource( return null; } -async function legacyKebabSourceWarning( - semanticLayerService: MetricflowSemanticLayerWriter, - connectionId: string, - modelRef: string, - sourceName: string, -): Promise { - const kebabName = toKebabCaseMetricflowName(modelRef); - if (kebabName === sourceName) { - return null; - } - const legacy = await semanticLayerService.loadSource(connectionId, kebabName); - if (!legacy) { - return null; - } - return ( - `MetricFlow sync: legacy kebab-case source '${kebabName}' still exists alongside the new source ` + - `'${sourceName}' (modelRef '${modelRef}'). Migrate persisted references before deleting the old file.` - ); -} - async function repairSourcesAfterPartialImportFailures(input: { semanticLayerService: MetricflowSemanticLayerWriter; connectionId: string; diff --git a/packages/context/src/ingest/ingest-bundle.runner.test.ts b/packages/context/src/ingest/ingest-bundle.runner.test.ts index 6134fbe7..b337a3f0 100644 --- a/packages/context/src/ingest/ingest-bundle.runner.test.ts +++ b/packages/context/src/ingest/ingest-bundle.runner.test.ts @@ -1518,7 +1518,6 @@ describe('IngestBundleRunner — Stages 1 → 7', () => { patternPagesWritten: 3, stalePatternPagesMarked: 1, archivedPatternPages: 1, - legacyPagesDeleted: 1, }, warnings: [], errors: [], @@ -1551,7 +1550,7 @@ describe('IngestBundleRunner — Stages 1 → 7', () => { expect(memoryFlow.snapshot().events).toContainEqual( expect.objectContaining({ type: 'saved', - wikiCount: 6, + wikiCount: 5, slCount: 3, }), ); diff --git a/packages/context/src/ingest/local-adapters.test.ts b/packages/context/src/ingest/local-adapters.test.ts index ad3b23f4..a962763d 100644 --- a/packages/context/src/ingest/local-adapters.test.ts +++ b/packages/context/src/ingest/local-adapters.test.ts @@ -105,7 +105,6 @@ describe('local ingest adapters', () => { return { headers: [], rows: [] }; }, }, - postgresBaselineRootDir: join(project.projectDir, '.ktx/cache/historic-sql'), }, }); @@ -181,9 +180,12 @@ describe('local ingest adapters', () => { historicSql: { enabled: true, dialect: 'postgres', - minCalls: 7, + minExecutions: 7, maxTemplatesPerRun: 123, - serviceAccountUserPatterns: ['^svc_'], + filters: { + serviceAccounts: { patterns: ['^svc_'], mode: 'exclude' }, + dropTrivialProbes: true, + }, }, }, }); @@ -385,7 +387,7 @@ describe('local ingest adapters', () => { connections: { 'prod-lookml': { driver: 'lookml', - repo_url: 'https://github.com/acme/looker.git', + repoUrl: 'https://github.com/acme/looker.git', branch: 'main', path: 'models', auth_token_ref: 'env:GITHUB_TOKEN', @@ -410,7 +412,7 @@ describe('local ingest adapters', () => { }); }); - it('rejects local LookML scheduled pulls when repo_url is missing', async () => { + it('rejects local LookML scheduled pulls when repoUrl is missing', async () => { const lookmlProject = { projectDir: tempDir, config: { connections: { 'prod-lookml': { driver: 'lookml' } } }, diff --git a/packages/context/src/ingest/local-adapters.ts b/packages/context/src/ingest/local-adapters.ts index 533bd526..0bf5fd42 100644 --- a/packages/context/src/ingest/local-adapters.ts +++ b/packages/context/src/ingest/local-adapters.ts @@ -50,7 +50,6 @@ export interface DefaultLocalIngestAdaptersOptions { reader?: HistoricSqlReader; queryClient?: unknown; postgresQueryClient?: KtxPostgresQueryClient; - postgresBaselineRootDir?: string; now?: () => Date; }; looker?: { @@ -129,7 +128,6 @@ export function createDefaultLocalIngestAdapters( sqlAnalysis: options.historicSql.sqlAnalysis, reader: options.historicSql.reader ?? new PostgresPgssReader(), queryClient, - legacyPostgresBaselineRootDir: options.historicSql.postgresBaselineRootDir, now: options.historicSql.now, }), ); @@ -163,11 +161,11 @@ function stringField(value: unknown): string | null { function localLookmlPullConfigFromConnection(connection: Record | undefined, env: NodeJS.ProcessEnv) { const mappings = isRecord(connection?.mappings) ? connection.mappings : {}; - const authTokenRef = stringField(connection?.auth_token_ref) ?? stringField(connection?.authTokenRef); - const literalAuthToken = stringField(connection?.authToken) ?? stringField(connection?.auth_token); + const authTokenRef = stringField(connection?.auth_token_ref); + const literalAuthToken = stringField(connection?.auth_token); return pullConfigFromIntegrationConfig({ - repoUrl: stringField(connection?.repoUrl) ?? stringField(connection?.repo_url) ?? null, + repoUrl: stringField(connection?.repoUrl) ?? null, branch: stringField(connection?.branch), path: stringField(connection?.path), authToken: literalAuthToken ?? resolveKtxConfigReference(authTokenRef ?? undefined, env) ?? null, @@ -176,27 +174,21 @@ function localLookmlPullConfigFromConnection(connection: Record } function localDbtPullConfigFromConnection(connection: Record | undefined, env: NodeJS.ProcessEnv) { - const sourceDir = stringField(connection?.source_dir) ?? stringField(connection?.sourceDir); - const repoUrl = stringField(connection?.repo_url) ?? stringField(connection?.repoUrl); + const sourceDir = stringField(connection?.source_dir); + const repoUrl = stringField(connection?.repo_url); if (sourceDir) { return { sourceDir, ...(stringField(connection?.profiles_path) ? { profilesPath: stringField(connection?.profiles_path) } : {}), - ...(stringField(connection?.profilesPath) ? { profilesPath: stringField(connection?.profilesPath) } : {}), ...(stringField(connection?.target) ? { target: stringField(connection?.target) } : {}), ...(stringField(connection?.project_name) ? { projectName: stringField(connection?.project_name) } : {}), - ...(stringField(connection?.projectName) ? { projectName: stringField(connection?.projectName) } : {}), }; } if (!repoUrl) { return undefined; } const authToken = - stringField(connection?.authToken) ?? - resolveKtxConfigReference( - stringField(connection?.auth_token_ref) ?? stringField(connection?.authTokenRef) ?? undefined, - env, - ); + stringField(connection?.auth_token) ?? resolveKtxConfigReference(stringField(connection?.auth_token_ref) ?? undefined, env); return { repoUrl, ...(stringField(connection?.branch) ? { branch: stringField(connection?.branch) } : {}), @@ -280,8 +272,8 @@ export async function localPullConfigForAdapter( ? (metricflow as Record) : null; const authToken = - typeof metricflowConfig?.authToken === 'string' - ? metricflowConfig.authToken + typeof metricflowConfig?.auth_token === 'string' + ? metricflowConfig.auth_token : resolveKtxConfigReference( typeof metricflowConfig?.auth_token_ref === 'string' ? metricflowConfig.auth_token_ref : undefined, options.looker?.env ?? process.env, diff --git a/packages/context/src/ingest/report-snapshot.test.ts b/packages/context/src/ingest/report-snapshot.test.ts index 13d3eff7..c949a3cc 100644 --- a/packages/context/src/ingest/report-snapshot.test.ts +++ b/packages/context/src/ingest/report-snapshot.test.ts @@ -22,7 +22,7 @@ function validReportSnapshot() { { target: 'wiki', type: 'created', key: 'knowledge/global/revenue.md', detail: 'Revenue overview' }, { target: 'sl', type: 'updated', key: 'warehouse.orders', detail: 'Added order amount measure' }, ], - touchedSlSources: ['warehouse.orders'], + touchedSlSources: [{ connectionId: 'warehouse', sourceName: 'orders' }], }, ], failedWorkUnits: [], @@ -106,7 +106,7 @@ describe('parseIngestReportSnapshot', () => { expect(snapshot.body.toolTranscripts).toHaveLength(1); }); - it('parses target-aware actions and normalizes legacy touched source strings', () => { + it('parses target-aware actions and touched source objects', () => { const report = validReportSnapshot(); report.body.workUnits[0] = { ...report.body.workUnits[0], @@ -119,8 +119,7 @@ describe('parseIngestReportSnapshot', () => { targetConnectionId: 'warehouse-1', }, ], - // Legacy report shape: bare strings are normalized to the report connection ID. - touchedSlSources: ['looker__b2b__sales_pipeline'], + touchedSlSources: [{ connectionId: 'warehouse-1', sourceName: 'looker__b2b__sales_pipeline' }], } as never; const snapshot = parseIngestReportSnapshot(report); @@ -135,7 +134,7 @@ describe('parseIngestReportSnapshot', () => { }, ]); expect(snapshot.body.workUnits[0]?.touchedSlSources).toEqual([ - { connectionId: 'warehouse', sourceName: 'looker__b2b__sales_pipeline' }, + { connectionId: 'warehouse-1', sourceName: 'looker__b2b__sales_pipeline' }, ]); }); diff --git a/packages/context/src/ingest/report-snapshot.ts b/packages/context/src/ingest/report-snapshot.ts index 76565ad9..de377dd5 100644 --- a/packages/context/src/ingest/report-snapshot.ts +++ b/packages/context/src/ingest/report-snapshot.ts @@ -1,5 +1,4 @@ import * as z from 'zod'; -import type { TouchedSlSource } from '../tools/index.js'; import { memoryFlowReplayInputSchema } from './memory-flow/schema.js'; import type { IngestReportSnapshot } from './reports.js'; @@ -24,8 +23,6 @@ const touchedSlSourceSchema = z.object({ sourceName: z.string().min(1), }); -const touchedSlSourceInputSchema = z.union([z.string(), touchedSlSourceSchema]); - const conflictResolvedSchema = z .object({ unitKey: z.string().optional(), @@ -42,7 +39,7 @@ const evictionAppliedSchema = z rawPath: z.string(), artifactKind: z.enum(['sl', 'wiki']), artifactKey: z.string(), - action: z.enum(['removed', 'retained_deprecated']), + action: z.literal('removed'), reason: z.string(), }) .passthrough(); @@ -147,7 +144,7 @@ export const ingestReportSnapshotSchema = z status: z.enum(['success', 'failed']), reason: z.string().optional(), actions: z.array(ingestActionSchema), - touchedSlSources: z.array(touchedSlSourceInputSchema), + touchedSlSources: z.array(touchedSlSourceSchema), slDisallowed: z.boolean().optional(), slDisallowedReason: z.enum(['lookml_connection_mismatch']).optional(), }), @@ -171,26 +168,10 @@ export const ingestReportSnapshotSchema = z }) .passthrough(); -function normalizeTouchedSlSources(connectionId: string, value: Array): TouchedSlSource[] { - return value.map((entry) => - typeof entry === 'string' - ? { connectionId, sourceName: entry } - : { connectionId: entry.connectionId, sourceName: entry.sourceName }, - ); -} - export function parseIngestReportSnapshot(value: unknown): IngestReportSnapshot { const result = ingestReportSnapshotSchema.safeParse(value); if (!result.success) { throw new Error(`Invalid ingest report snapshot: ${z.prettifyError(result.error)}`); } - const snapshot = result.data as IngestReportSnapshot; - snapshot.body.workUnits = snapshot.body.workUnits.map((workUnit) => ({ - ...workUnit, - touchedSlSources: normalizeTouchedSlSources( - snapshot.connectionId, - workUnit.touchedSlSources as Array, - ), - })); - return snapshot; + return result.data as IngestReportSnapshot; } diff --git a/packages/context/src/ingest/reports.ts b/packages/context/src/ingest/reports.ts index cda4d7c1..672c5bfb 100644 --- a/packages/context/src/ingest/reports.ts +++ b/packages/context/src/ingest/reports.ts @@ -111,8 +111,7 @@ export function postProcessorSavedMemoryCounts( wikiCount: numericResultField(record, 'patternPagesWritten') + numericResultField(record, 'stalePatternPagesMarked') + - numericResultField(record, 'archivedPatternPages') + - numericResultField(record, 'legacyPagesDeleted'), + numericResultField(record, 'archivedPatternPages'), slCount: numericResultField(record, 'tableUsageMerged') + numericResultField(record, 'staleTablesMarked'), }; } diff --git a/packages/context/src/ingest/stages/stage-index.types.ts b/packages/context/src/ingest/stages/stage-index.types.ts index c8d7e4b3..7de26bc8 100644 --- a/packages/context/src/ingest/stages/stage-index.types.ts +++ b/packages/context/src/ingest/stages/stage-index.types.ts @@ -25,7 +25,7 @@ export interface EvictionAppliedRecord { rawPath: string; artifactKind: 'sl' | 'wiki'; artifactKey: string; - action: 'removed' | 'retained_deprecated'; + action: 'removed'; reason: string; } diff --git a/packages/context/src/ingest/tools/emit-eviction-decision.tool.ts b/packages/context/src/ingest/tools/emit-eviction-decision.tool.ts index f44214ea..28a32a5b 100644 --- a/packages/context/src/ingest/tools/emit-eviction-decision.tool.ts +++ b/packages/context/src/ingest/tools/emit-eviction-decision.tool.ts @@ -22,7 +22,7 @@ export function createEmitEvictionDecisionTool(deps: EmitEvictionDecisionDeps) { rawPath: z.string().min(1), artifactKind: z.enum(['sl', 'wiki']), artifactKey: z.string().min(1), - action: z.enum(['removed', 'retained_deprecated']), + action: z.literal('removed'), reason: z.string().min(1), }), execute: async (input): Promise => { diff --git a/packages/context/src/ingest/tools/emit-reconciliation-records.tool.test.ts b/packages/context/src/ingest/tools/emit-reconciliation-records.tool.test.ts index a3e7b34f..9178c989 100644 --- a/packages/context/src/ingest/tools/emit-reconciliation-records.tool.test.ts +++ b/packages/context/src/ingest/tools/emit-reconciliation-records.tool.test.ts @@ -88,14 +88,14 @@ describe('reconciliation emit tools', () => { await executeTool(tool, { rawPath: 'views/old_orders.view.lkml', artifactKind: 'wiki', - artifactKey: 'orders/legacy', - action: 'retained_deprecated', + artifactKey: 'orders/old', + action: 'removed', reason: 'first pass', }); await executeTool(tool, { rawPath: 'views/old_orders.view.lkml', artifactKind: 'wiki', - artifactKey: 'orders/legacy', + artifactKey: 'orders/old', action: 'removed', reason: 'second pass after checking references', }); @@ -104,7 +104,7 @@ describe('reconciliation emit tools', () => { { rawPath: 'views/old_orders.view.lkml', artifactKind: 'wiki', - artifactKey: 'orders/legacy', + artifactKey: 'orders/old', action: 'removed', reason: 'second pass after checking references', }, diff --git a/packages/context/src/ingest/tools/eviction-list.tool.ts b/packages/context/src/ingest/tools/eviction-list.tool.ts index 1e2ca3a0..4ed08d63 100644 --- a/packages/context/src/ingest/tools/eviction-list.tool.ts +++ b/packages/context/src/ingest/tools/eviction-list.tool.ts @@ -12,7 +12,7 @@ export interface EvictionListDeps { export function createEvictionListTool(deps: EvictionListDeps) { return tool({ description: - 'List every artifact that the most recent completed sync produced from a now-deleted raw file. Use this to decide whether to remove (no inbound refs) or retain with deprecation (has inbound refs). Inbound refs are NOT currently computed — treat every retained entry as a candidate and ask the user via the IngestReport. After deciding, record the decision with context_eviction_decision_write so the ingest report lists every deleted-source decision.', + 'List every artifact that the most recent completed sync produced from a now-deleted raw file. Remove each listed artifact and record the decision with context_eviction_decision_write so the ingest report lists every deleted-source decision.', inputSchema: z.object({}), execute: async () => { if (deps.deletedRawPaths.length === 0) { diff --git a/packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts b/packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts index a1edf807..b916107c 100644 --- a/packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts +++ b/packages/context/src/ingest/tools/warehouse-verification/warehouse-catalog.service.ts @@ -88,9 +88,8 @@ interface ConnectionCatalog { } type TableWithDescriptions = KtxSchemaTable & { - description?: string | null; descriptions?: Record; - columns: Array }>; + columns: Array }>; }; function normalize(value: string | null | undefined): string { @@ -220,14 +219,14 @@ function matchedOnTable(table: TableWithDescriptions, query: string): RawSchemaH if (normalize(table.comment).includes(q)) { return 'comment'; } - if (normalize(firstDescription(table.descriptions) ?? table.description).includes(q)) { + if (normalize(firstDescription(table.descriptions)).includes(q)) { return 'description'; } return null; } function matchedOnColumn( - column: KtxSchemaColumn & { description?: string | null; descriptions?: Record }, + column: KtxSchemaColumn & { descriptions?: Record }, query: string, ): 'name' | 'comment' | 'description' | null { const q = normalize(query); @@ -240,7 +239,7 @@ function matchedOnColumn( if (normalize(column.comment).includes(q)) { return 'comment'; } - if (normalize(firstDescription(column.descriptions) ?? column.description).includes(q)) { + if (normalize(firstDescription(column.descriptions)).includes(q)) { return 'description'; } return null; @@ -285,13 +284,10 @@ export class WarehouseCatalogService { display: formatDisplay(catalog.driver, table), kind: table.kind, comment: table.comment, - description: table.description ?? firstDescription(table.descriptions), + description: firstDescription(table.descriptions), rowCount: profileTable?.rowCount ?? table.estimatedRows ?? null, columns: table.columns.map((rawColumn) => { - const column = rawColumn as KtxSchemaColumn & { - description?: string | null; - descriptions?: Record; - }; + const column = rawColumn as KtxSchemaColumn & { descriptions?: Record }; const profileColumn = profileColumns[columnKey(table, column.name)] ?? Object.entries(profileColumns).find( diff --git a/packages/context/src/ingest/wiki-sl-ref-repair.test.ts b/packages/context/src/ingest/wiki-sl-ref-repair.test.ts index 958386c7..68f2b349 100644 --- a/packages/context/src/ingest/wiki-sl-ref-repair.test.ts +++ b/packages/context/src/ingest/wiki-sl-ref-repair.test.ts @@ -40,7 +40,7 @@ describe('repairWikiSlRefs', () => { }; const configService = { listFiles: vi.fn(async () => ({ - files: ['global/accounts-at-risk.md', 'global/historic-sql/nested-legacy.md'], + files: ['global/accounts-at-risk.md', 'global/historic-sql/nested-old.md'], })), }; const semanticLayerService = { diff --git a/packages/context/src/mcp/local-project-ports.test.ts b/packages/context/src/mcp/local-project-ports.test.ts index 85a3c2c7..e3812960 100644 --- a/packages/context/src/mcp/local-project-ports.test.ts +++ b/packages/context/src/mcp/local-project-ports.test.ts @@ -576,7 +576,8 @@ describe('createLocalProjectMcpContextPorts', () => { sourceName: 'orders', yaml: [ 'name: orders', - 'description: Revenue facts', + 'descriptions:', + ' user: Revenue facts', 'table: public.orders', 'grain:', ' - order_id', diff --git a/packages/context/src/memory/capture-signals.ts b/packages/context/src/memory/capture-signals.ts index 8860474a..856df30b 100644 --- a/packages/context/src/memory/capture-signals.ts +++ b/packages/context/src/memory/capture-signals.ts @@ -114,7 +114,6 @@ export function stepBudgetFor(sourceType: MemoryAgentSourceType): number { case 'external_ingest': return 30; case 'backfill': - case 'sql-review-migration': return 25; } } @@ -122,7 +121,7 @@ export function stepBudgetFor(sourceType: MemoryAgentSourceType): number { export function promptNameFor(sourceType: MemoryAgentSourceType): string { return sourceType === 'external_ingest' ? 'memory_agent_external_ingest' - : sourceType === 'backfill' || sourceType === 'sql-review-migration' + : sourceType === 'backfill' ? 'memory_agent_backfill' : 'memory_agent_research'; } diff --git a/packages/context/src/memory/types.ts b/packages/context/src/memory/types.ts index aa50cd8c..207eb238 100644 --- a/packages/context/src/memory/types.ts +++ b/packages/context/src/memory/types.ts @@ -16,7 +16,7 @@ import type { import type { ToolContext, ToolSession, TouchedSlSourceSet } from '../tools/index.js'; import type { KnowledgeIndexPort, KnowledgeWikiService } from '../wiki/index.js'; -export type MemoryAgentSourceType = 'research' | 'external_ingest' | 'backfill' | 'sql-review-migration'; +export type MemoryAgentSourceType = 'research' | 'external_ingest' | 'backfill'; export interface MemoryAgentInput { userId: string; diff --git a/packages/context/src/project/config.test.ts b/packages/context/src/project/config.test.ts index cad7945c..5f13729e 100644 --- a/packages/context/src/project/config.test.ts +++ b/packages/context/src/project/config.test.ts @@ -216,15 +216,15 @@ project: demo scan: relationships: enabled: false - llm_proposals: false - validation_required_for_manifest: true - accept_threshold: 0.91 - review_threshold: 0.61 - max_llm_tables_per_batch: 12 - max_candidates_per_column: 7 - profile_sample_rows: 500 - validation_concurrency: 2 - validation_budget: 0 + llmProposals: false + validationRequiredForManifest: true + acceptThreshold: 0.91 + reviewThreshold: 0.61 + maxLlmTablesPerBatch: 12 + maxCandidatesPerColumn: 7 + profileSampleRows: 500 + validationConcurrency: 2 + validationBudget: 0 `); expect(config.scan.relationships).toEqual({ @@ -256,7 +256,7 @@ scan: project: demo scan: relationships: - validation_budget: all + validationBudget: all `); expect(config.scan.relationships.validationBudget).toBe('all'); @@ -268,13 +268,13 @@ scan: project: demo scan: relationships: - accept_threshold: 2 - review_threshold: -1 - max_llm_tables_per_batch: 0 - max_candidates_per_column: -4 - profile_sample_rows: 0 - validation_concurrency: 0 - validation_budget: 1.5 + acceptThreshold: 2 + reviewThreshold: -1 + maxLlmTablesPerBatch: 0 + maxCandidatesPerColumn: -4 + profileSampleRows: 0 + validationConcurrency: 0 + validationBudget: 1.5 `); expect(config.scan.relationships).toMatchObject({ @@ -293,13 +293,13 @@ scan: project: demo scan: relationships: - validation_budget: infinite + validationBudget: infinite `); expect(config.scan.relationships).not.toHaveProperty('validationBudget'); }); - it('rejects legacy local LLM and embedding fields', () => { + it('rejects unsupported local LLM and embedding fields', () => { expect(() => parseKtxProjectConfig(` project: demo diff --git a/packages/context/src/project/config.ts b/packages/context/src/project/config.ts index 5da193f2..b00d0148 100644 --- a/packages/context/src/project/config.ts +++ b/packages/context/src/project/config.ts @@ -211,7 +211,7 @@ function scanEnrichmentMode(value: unknown, fallback: KtxScanEnrichmentMode): Kt throw new Error(`Unsupported scan.enrichment.mode: ${String(value)}`); } -function rejectLegacyProvider(section: string, value: unknown): void { +function rejectUnsupportedProvider(section: string, value: unknown): void { if (value !== undefined) { throw new Error(`Unsupported ${section}.provider: use ${section}.backend`); } @@ -276,7 +276,7 @@ function parseProjectLlmProviderConfig( defaults: KtxProjectLlmProviderConfig, section: string, ): KtxProjectLlmProviderConfig { - rejectLegacyProvider(section, raw.provider); + rejectUnsupportedProvider(section, raw.provider); const vertex = isRecord(raw.vertex) ? { @@ -309,7 +309,7 @@ function parseProjectEmbeddingConfig( defaults: KtxProjectEmbeddingConfig, section: string, ): KtxProjectEmbeddingConfig { - rejectLegacyProvider(section, raw.provider); + rejectUnsupportedProvider(section, raw.provider); const openai = optionalProviderConfig(raw.openai); const sentenceTransformers = isRecord(raw.sentenceTransformers) @@ -339,36 +339,21 @@ function parseScanRelationshipConfig( raw: Record, defaults: KtxScanRelationshipConfig, ): KtxScanRelationshipConfig { - const validationBudget = validationBudgetConfigValue( - raw.validation_budget ?? raw.validationBudget, - defaults.validationBudget, - ); + const validationBudget = validationBudgetConfigValue(raw.validationBudget, defaults.validationBudget); return { enabled: booleanValue(raw.enabled, defaults.enabled), - llmProposals: booleanValue(raw.llm_proposals ?? raw.llmProposals, defaults.llmProposals), + llmProposals: booleanValue(raw.llmProposals, defaults.llmProposals), validationRequiredForManifest: booleanValue( - raw.validation_required_for_manifest ?? raw.validationRequiredForManifest, + raw.validationRequiredForManifest, defaults.validationRequiredForManifest, ), - acceptThreshold: ratioConfigValue(raw.accept_threshold ?? raw.acceptThreshold, defaults.acceptThreshold), - reviewThreshold: ratioConfigValue(raw.review_threshold ?? raw.reviewThreshold, defaults.reviewThreshold), - maxLlmTablesPerBatch: positiveIntegerConfigValue( - raw.max_llm_tables_per_batch ?? raw.maxLlmTablesPerBatch, - defaults.maxLlmTablesPerBatch, - ), - maxCandidatesPerColumn: positiveIntegerConfigValue( - raw.max_candidates_per_column ?? raw.maxCandidatesPerColumn, - defaults.maxCandidatesPerColumn, - ), - profileSampleRows: positiveIntegerConfigValue( - raw.profile_sample_rows ?? raw.profileSampleRows, - defaults.profileSampleRows, - ), - validationConcurrency: positiveIntegerConfigValue( - raw.validation_concurrency ?? raw.validationConcurrency, - defaults.validationConcurrency, - ), + acceptThreshold: ratioConfigValue(raw.acceptThreshold, defaults.acceptThreshold), + reviewThreshold: ratioConfigValue(raw.reviewThreshold, defaults.reviewThreshold), + maxLlmTablesPerBatch: positiveIntegerConfigValue(raw.maxLlmTablesPerBatch, defaults.maxLlmTablesPerBatch), + maxCandidatesPerColumn: positiveIntegerConfigValue(raw.maxCandidatesPerColumn, defaults.maxCandidatesPerColumn), + profileSampleRows: positiveIntegerConfigValue(raw.profileSampleRows, defaults.profileSampleRows), + validationConcurrency: positiveIntegerConfigValue(raw.validationConcurrency, defaults.validationConcurrency), ...(validationBudget !== undefined ? { validationBudget } : {}), }; } diff --git a/packages/context/src/scan/relationship-benchmarks.test.ts b/packages/context/src/scan/relationship-benchmarks.test.ts index b4e5c782..aff025aa 100644 --- a/packages/context/src/scan/relationship-benchmarks.test.ts +++ b/packages/context/src/scan/relationship-benchmarks.test.ts @@ -33,7 +33,7 @@ const EXPECTED_LINKS: KtxRelationshipBenchmarkExpectedLinks = { }; const CHECKED_IN_FIXTURE_ORIGINS = { - abbreviated_legacy_no_declared_constraints: 'synthetic', + abbreviated_old_no_declared_constraints: 'synthetic', adventureworks_oltp_with_declared_metadata: 'public', adventureworkslt_with_declared_metadata: 'public', analytical_warehouse_no_naming_convention: 'synthetic', @@ -606,7 +606,7 @@ describe('relationship benchmarks', () => { const byId = new Map(fixtures.map((fixture) => [fixture.id, fixture])); const adversarialIds = [ 'non_english_naming_no_declared_constraints', - 'abbreviated_legacy_no_declared_constraints', + 'abbreviated_old_no_declared_constraints', 'analytical_warehouse_no_naming_convention', 'mixed_case_within_schema_no_declared_constraints', 'polymorphic_partial_overlap_no_declared_constraints', diff --git a/packages/context/src/scan/relationship-diagnostics.test.ts b/packages/context/src/scan/relationship-diagnostics.test.ts index 3f3bad1b..7c1dbb76 100644 --- a/packages/context/src/scan/relationship-diagnostics.test.ts +++ b/packages/context/src/scan/relationship-diagnostics.test.ts @@ -141,7 +141,7 @@ describe('relationship diagnostics artifacts', () => { ); }); - it('adapts legacy relationship updates into the richer artifact shape', () => { + it('adapts relationship updates into the artifact shape', () => { const artifacts = buildKtxRelationshipArtifacts({ connectionId: 'warehouse', relationshipUpdate: { diff --git a/packages/context/src/search/backend-conformance.test.ts b/packages/context/src/search/backend-conformance.test.ts index d2d8e3bf..95858486 100644 --- a/packages/context/src/search/backend-conformance.test.ts +++ b/packages/context/src/search/backend-conformance.test.ts @@ -45,7 +45,8 @@ const ORDERS_YAML = [ const FINANCE_ORDERS_YAML = [ 'name: orders', - 'description: Finance orders used for invoice reconciliation.', + 'descriptions:', + ' user: Finance orders used for invoice reconciliation.', 'table: finance.orders', 'grain:', ' - order_id', diff --git a/packages/context/src/sl/description-normalization.ts b/packages/context/src/sl/description-normalization.ts index 5a1b5ab6..ef657fdd 100644 --- a/packages/context/src/sl/description-normalization.ts +++ b/packages/context/src/sl/description-normalization.ts @@ -28,16 +28,11 @@ function hasDescriptions(descriptions: DescriptionMap): boolean { function withDescriptionMap(record: Record, fallback: string | null): Record { const descriptions = cleanDescriptionMap(record.descriptions); - const flatDescription = cleanText(record.description); - if (flatDescription && !descriptions.user) { - descriptions.user = flatDescription; - } if (!hasDescriptions(descriptions) && fallback) { descriptions.ktx = fallback; } const next = { ...record }; - delete next.description; if (hasDescriptions(descriptions)) { next.descriptions = descriptions; } else { diff --git a/packages/context/src/sl/local-sl.test.ts b/packages/context/src/sl/local-sl.test.ts index aa48546b..b7d56e22 100644 --- a/packages/context/src/sl/local-sl.test.ts +++ b/packages/context/src/sl/local-sl.test.ts @@ -29,7 +29,8 @@ const ORDERS_YAML = [ const SUPPORT_YAML = [ 'name: tickets', - 'description: Support tickets grouped by priority.', + 'descriptions:', + ' user: Support tickets grouped by priority.', 'table: public.tickets', 'grain:', ' - ticket_id', @@ -278,7 +279,8 @@ describe('local semantic-layer helpers', () => { sourceName: 'orders', yaml: [ 'name: orders', - 'description: Finance orders used for invoice reconciliation.', + 'descriptions:', + ' user: Finance orders used for invoice reconciliation.', 'table: finance.orders', 'grain:', ' - order_id', diff --git a/packages/context/src/sl/pglite-sl-search-prototype.test.ts b/packages/context/src/sl/pglite-sl-search-prototype.test.ts index 1d0ece25..0c599dca 100644 --- a/packages/context/src/sl/pglite-sl-search-prototype.test.ts +++ b/packages/context/src/sl/pglite-sl-search-prototype.test.ts @@ -10,7 +10,8 @@ import { searchLocalSlSourcesWithPglitePrototype } from './pglite-sl-search-prot const ORDERS_YAML = [ 'name: orders', - 'description: Orders with paid revenue and refund status.', + 'descriptions:', + ' user: Orders with paid revenue and refund status.', 'table: public.orders', 'grain:', ' - order_id', @@ -29,7 +30,8 @@ const ORDERS_YAML = [ const FINANCE_ORDERS_YAML = [ 'name: orders', - 'description: Finance orders used for invoice reconciliation.', + 'descriptions:', + ' user: Finance orders used for invoice reconciliation.', 'table: finance.orders', 'grain:', ' - order_id', @@ -43,7 +45,8 @@ const FINANCE_ORDERS_YAML = [ const CUSTOMERS_YAML = [ 'name: customers', - 'description: Customer lifecycle accounts by region.', + 'descriptions:', + ' user: Customer lifecycle accounts by region.', 'table: public.customers', 'grain:', ' - customer_id', diff --git a/packages/context/src/sl/schemas.ts b/packages/context/src/sl/schemas.ts index a42ecc87..a57359d4 100644 --- a/packages/context/src/sl/schemas.ts +++ b/packages/context/src/sl/schemas.ts @@ -80,14 +80,13 @@ const joinDeclarationSchema = z.object({ const sourceColumnSchema = z.object({ name: unqualifiedNameSchema, - // type/description optional on standalone sources: compose-time enrichment fills them + // type/descriptions optional on standalone sources: compose-time enrichment fills them // from the manifest entry named in `inherits_columns_from`. If the agent does not set // `inherits_columns_from`, or the column is not in the manifest, type must be present // — surfaced by sl_validate. type: z.enum(columnTypeValues).optional(), role: z.enum(columnRoleValues).optional(), visibility: z.enum(columnVisibilityValues).optional(), - description: z.string().optional(), descriptions: descriptionsSchema.optional(), expr: z.string().optional(), constraints: sourceKeyedColumnConstraintsSchema.optional(), @@ -102,7 +101,6 @@ const overlayColumnSchema = z type: z.enum(columnTypeValues).optional(), role: z.enum(columnRoleValues).optional(), visibility: z.enum(columnVisibilityValues).optional(), - description: z.string().optional(), descriptions: descriptionsSchema.optional(), expr: z.string().optional(), }) @@ -114,7 +112,6 @@ const overlayColumnSchema = z export const sourceDefinitionSchema = z .object({ name: z.string().min(1), - description: z.string().optional(), descriptions: descriptionsSchema.optional(), // Accepted for documentation parity with the Python spec; behavior is driven // by the `table` / `sql` fields, not by this discriminator. @@ -150,7 +147,6 @@ export const sourceDefinitionSchema = z export const sourceOverlaySchema = z .object({ name: z.string().min(1), - description: z.string().optional(), descriptions: z.record(z.string(), z.string()).optional(), grain: z.array(unqualifiedNameSchema).optional(), columns: z.array(overlayColumnSchema).optional(), diff --git a/packages/context/src/sl/semantic-layer.service.test.ts b/packages/context/src/sl/semantic-layer.service.test.ts index 308cc5aa..179904d5 100644 --- a/packages/context/src/sl/semantic-layer.service.test.ts +++ b/packages/context/src/sl/semantic-layer.service.test.ts @@ -98,7 +98,7 @@ describe('composeOverlay', () => { ...baseTable, segments: [{ name: 'pre_existing', expr: 'is_paid = true' }], }; - const overlay = { name: 'fct_labs', description: 'no segments here' }; + const overlay = { name: 'fct_labs', descriptions: { user: 'no segments here' } }; const composed = composeOverlay(baseWithSegments, overlay); expect(composed.segments).toEqual([{ name: 'pre_existing', expr: 'is_paid = true' }]); }); @@ -128,7 +128,7 @@ describe('composeOverlay', () => { it('still handles existing known keys without regression', () => { const overlay = { name: 'fct_labs', - description: 'patient lab orders', + descriptions: { user: 'patient lab orders' }, exclude_columns: ['admin_user_id'], columns: [{ name: 'is_byol', type: 'boolean', expr: "lab_type = 'byol'" }], measures: [{ name: 'count_all', expr: 'count(*)' }], @@ -675,19 +675,21 @@ describe('loadAllSources — standalone enrichment via inherits_columns_from', ( expect(aav?.columns).toEqual([{ name: 'FOO', type: 'string' }]); }); - it('normalizes legacy flat source and column descriptions when loading standalone files', async () => { + it('loads standalone source and column description maps', async () => { const standalonePath = 'semantic-layer/conn-1/orders.yaml'; configService.listFiles.mockResolvedValue({ files: [standalonePath] }); configService.readFile.mockResolvedValue({ content: [ 'name: orders', - 'description: Finance orders used for invoice reconciliation.', + 'descriptions:', + ' user: Finance orders used for invoice reconciliation.', 'table: public.orders', 'grain: [id]', 'columns:', ' - name: id', ' type: string', - ' description: Stable order identifier.', + ' descriptions:', + ' user: Stable order identifier.', ].join('\n'), }); diff --git a/packages/context/src/sl/semantic-layer.service.ts b/packages/context/src/sl/semantic-layer.service.ts index 0616851d..7d13d10a 100644 --- a/packages/context/src/sl/semantic-layer.service.ts +++ b/packages/context/src/sl/semantic-layer.service.ts @@ -113,7 +113,7 @@ export class SemanticLayerService { `standalone source '${source.name}' shadows an existing manifest entry and ` + `will drop the manifest's columns and joins. Rewrite as an overlay: remove ` + `"sql:", "table:", "grain:", "columns:", "joins:"; keep only "name:" plus ` + - `"measures:"/"segments:"/"description:"`; + `"measures:"/"segments:"/"descriptions:"`; warnings.push(msg); this.logger.warn(`[writeSource] ${msg}. Saving anyway.`); } @@ -935,16 +935,12 @@ export class SemanticLayerService { string, { descriptions?: Record; - description?: string; - db_description?: string; columns?: Array<{ name: string; type: string; pk?: boolean; nullable?: boolean; descriptions?: Record; - description?: string; - db_description?: string; }>; } >; @@ -952,12 +948,12 @@ export class SemanticLayerService { if (shard?.tables) { for (const [tableName, entry] of Object.entries(shard.tables)) { tables.set(tableName, { - descriptions: migrateDescriptions(entry.descriptions, entry.description, entry.db_description) ?? {}, + descriptions: entry.descriptions ?? {}, }); for (const col of entry.columns ?? []) { columns.set(`${tableName}.${col.name}`, { type: col.type, - descriptions: migrateDescriptions(col.descriptions, col.description, col.db_description) ?? {}, + descriptions: col.descriptions ?? {}, nullable: col.nullable, pk: col.pk, }); @@ -1055,11 +1051,7 @@ interface ManifestColumnEntry { type: string; pk?: boolean; nullable?: boolean; - // New format: descriptions map descriptions?: Record; - // Legacy format: flat fields (read-only backwards compat) - description?: string; - db_description?: string; constraints?: { dbt?: { not_null?: boolean; unique?: boolean } }; enum_values?: { dbt?: string[] }; tests?: { @@ -1077,11 +1069,7 @@ interface ManifestJoinEntry { export interface ManifestTableEntry { table: string; - // New format: descriptions map descriptions?: Record; - // Legacy format: flat fields (read-only backwards compat) - description?: string; - db_description?: string; columns: ManifestColumnEntry[]; joins?: ManifestJoinEntry[]; tags?: { dbt?: string[] }; @@ -1089,31 +1077,12 @@ export interface ManifestTableEntry { usage?: TableUsageOutput; } -/** Migrate legacy flat description/db_description fields to a descriptions map. */ -function migrateDescriptions( - descriptions?: Record, - description?: string, - dbDescription?: string, -): Record | undefined { - if (descriptions && Object.keys(descriptions).length > 0) { - return descriptions; - } - const result: Record = {}; - if (description) { - result.ai = description; - } - if (dbDescription) { - result.db = dbDescription; - } - return Object.keys(result).length > 0 ? result : undefined; -} - export function projectManifestEntry(name: string, entry: ManifestTableEntry): SemanticLayerSource { const columns = entry.columns.map((c) => ({ name: c.name, type: c.type, role: c.type === 'time' ? 'time' : undefined, - descriptions: migrateDescriptions(c.descriptions, c.description, c.db_description), + descriptions: c.descriptions, constraints: c.constraints, enum_values: c.enum_values, tests: c.tests, @@ -1126,7 +1095,7 @@ export function projectManifestEntry(name: string, entry: ManifestTableEntry): S return { name, table: entry.table, - descriptions: migrateDescriptions(entry.descriptions, entry.description, entry.db_description), + descriptions: entry.descriptions, grain, columns, joins: (entry.joins ?? []).map((j) => ({ to: j.to, on: j.on, relationship: j.relationship, source: j.source })), @@ -1359,7 +1328,6 @@ export function findDanglingSegmentRefs(source: Record): string const COMPOSE_KNOWN_KEYS = new Set([ 'name', - 'description', 'descriptions', 'grain', 'columns', diff --git a/packages/context/src/sl/tools/sl-edit-source.tool.ts b/packages/context/src/sl/tools/sl-edit-source.tool.ts index 27b582d5..30972707 100644 --- a/packages/context/src/sl/tools/sl-edit-source.tool.ts +++ b/packages/context/src/sl/tools/sl-edit-source.tool.ts @@ -127,7 +127,7 @@ If no source exists yet, use sl_write_source instead — this tool will reject t ` - name: `, ` expr: ""`, ` description: ""`, - `Overlay shape: "name:" plus any of "measures:", "segments:", "description:". Do NOT include "sql:", "table:", "grain:", "columns:", or "joins:" — those are inherited from the manifest.`, + `Overlay shape: "name:" plus any of "measures:", "segments:", "descriptions:". Do NOT include "sql:", "table:", "grain:", "columns:", or "joins:" — those are inherited from the manifest.`, ].join('\n'), ], sourceName, diff --git a/packages/context/src/sl/tools/sl-warehouse-validation.ts b/packages/context/src/sl/tools/sl-warehouse-validation.ts index a200dad9..e0d48721 100644 --- a/packages/context/src/sl/tools/sl-warehouse-validation.ts +++ b/packages/context/src/sl/tools/sl-warehouse-validation.ts @@ -89,7 +89,7 @@ export async function validateSingleSource( `${sourceName}.yaml: standalone source shadows an existing manifest entry — ` + `writing it as-is drops the manifest's columns and joins. ` + `Remove "sql:", "table:", "grain:", "columns:", and "joins:" and keep only ` + - `"name:" plus "measures:"/"segments:"/"description:" to write an overlay ` + + `"name:" plus "measures:"/"segments:"/"descriptions:" to write an overlay ` + `that inherits the manifest schema. Call sl_read_source to inspect the existing source first.`, ); return { errors, warnings }; diff --git a/packages/context/src/sl/tools/sl-write-source.tool.test.ts b/packages/context/src/sl/tools/sl-write-source.tool.test.ts index 1502c177..d9c58225 100644 --- a/packages/context/src/sl/tools/sl-write-source.tool.test.ts +++ b/packages/context/src/sl/tools/sl-write-source.tool.test.ts @@ -176,7 +176,7 @@ describe('SlWriteSourceTool — session gating', () => { expect((session.semanticLayerService as any).writeSource).toHaveBeenCalled(); }); - it('normalizes flat source and column descriptions before writing', async () => { + it('writes source and column description maps', async () => { const { tool, semanticLayerService } = makeTool(); const result = await tool.call( { @@ -184,10 +184,10 @@ describe('SlWriteSourceTool — session gating', () => { sourceName: 'orders', source: { name: 'orders', - description: 'Finance orders used for invoice reconciliation.', + descriptions: { user: 'Finance orders used for invoice reconciliation.' }, table: 'public.orders', grain: ['id'], - columns: [{ name: 'id', type: 'string', description: 'Stable order identifier.' }], + columns: [{ name: 'id', type: 'string', descriptions: { user: 'Stable order identifier.' } }], measures: [], joins: [], } as any, diff --git a/packages/context/src/sl/tools/sl-write-source.tool.ts b/packages/context/src/sl/tools/sl-write-source.tool.ts index 34b6f8c4..e7efb357 100644 --- a/packages/context/src/sl/tools/sl-write-source.tool.ts +++ b/packages/context/src/sl/tools/sl-write-source.tool.ts @@ -318,7 +318,7 @@ Do NOT join back to a table that the SQL already aggregates from if the grain co ` Writing standalone would drop the manifest's columns and joins, leaving only what you list here.`, `To add measures/segments on top of the manifest, rewrite this YAML as an overlay:`, ` - Remove "sql:", "table:", "grain:", "columns:", and "joins:".`, - ` - Keep only "name:", plus "measures:", "segments:", and/or "description:".`, + ` - Keep only "name:", plus "measures:", "segments:", and/or "descriptions:".`, ` - The manifest's schema is inherited automatically.`, `If you really need a different base table, use a different source name.`, ].join('\n'); diff --git a/packages/context/src/tools/tool-session.ts b/packages/context/src/tools/tool-session.ts index 023a8c8e..05da85d9 100644 --- a/packages/context/src/tools/tool-session.ts +++ b/packages/context/src/tools/tool-session.ts @@ -23,7 +23,7 @@ interface EvictionDecisionRecord { rawPath: string; artifactKind: 'wiki' | 'sl'; artifactKey: string; - action: 'removed' | 'retained_deprecated' | 'retained_supported'; + action: 'removed'; reason: string; } diff --git a/packages/context/src/wiki/local-knowledge.test.ts b/packages/context/src/wiki/local-knowledge.test.ts index 5ad66eb1..54bd3771 100644 --- a/packages/context/src/wiki/local-knowledge.test.ts +++ b/packages/context/src/wiki/local-knowledge.test.ts @@ -245,29 +245,4 @@ describe('local knowledge helpers', () => { ).rejects.toThrow('Invalid wiki key "orbit/company-overview". Wiki keys must be flat; use "orbit-company-overview".'); }); - it('ignores nested historic-SQL legacy paths when listing local knowledge pages', async () => { - await writeLocalKnowledgePage(project, { - key: 'historic-sql-paid-orders', - scope: 'GLOBAL', - summary: 'Flat historic SQL page', - content: 'Flat page body.', - tags: ['historic-sql'], - }); - await project.fileStore.writeFile( - 'knowledge/global/historic-sql/paid-orders.md', - '---\nsummary: Nested historic SQL page\nusage_mode: auto\n---\n\nNested body\n', - 'Test', - 'test@example.com', - 'Write nested legacy page', - ); - - await expect(listLocalKnowledgePages(project, { userId: 'local' })).resolves.toEqual([ - { - key: 'historic-sql-paid-orders', - path: 'knowledge/global/historic-sql-paid-orders.md', - scope: 'GLOBAL', - summary: 'Flat historic SQL page', - }, - ]); - }); }); diff --git a/packages/context/test/fixtures/relationship-benchmarks/abbreviated_legacy_no_declared_constraints/data.sqlite b/packages/context/test/fixtures/relationship-benchmarks/abbreviated_old_no_declared_constraints/data.sqlite similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/abbreviated_legacy_no_declared_constraints/data.sqlite rename to packages/context/test/fixtures/relationship-benchmarks/abbreviated_old_no_declared_constraints/data.sqlite diff --git a/packages/context/test/fixtures/relationship-benchmarks/abbreviated_legacy_no_declared_constraints/expected-links.yaml b/packages/context/test/fixtures/relationship-benchmarks/abbreviated_old_no_declared_constraints/expected-links.yaml similarity index 100% rename from packages/context/test/fixtures/relationship-benchmarks/abbreviated_legacy_no_declared_constraints/expected-links.yaml rename to packages/context/test/fixtures/relationship-benchmarks/abbreviated_old_no_declared_constraints/expected-links.yaml diff --git a/packages/context/test/fixtures/relationship-benchmarks/abbreviated_legacy_no_declared_constraints/fixture.yaml b/packages/context/test/fixtures/relationship-benchmarks/abbreviated_old_no_declared_constraints/fixture.yaml similarity index 50% rename from packages/context/test/fixtures/relationship-benchmarks/abbreviated_legacy_no_declared_constraints/fixture.yaml rename to packages/context/test/fixtures/relationship-benchmarks/abbreviated_old_no_declared_constraints/fixture.yaml index 275a1008..6a9b3810 100644 --- a/packages/context/test/fixtures/relationship-benchmarks/abbreviated_legacy_no_declared_constraints/fixture.yaml +++ b/packages/context/test/fixtures/relationship-benchmarks/abbreviated_old_no_declared_constraints/fixture.yaml @@ -1,5 +1,5 @@ -id: abbreviated_legacy_no_declared_constraints -name: Abbreviated legacy naming fixture with no declared constraints +id: abbreviated_old_no_declared_constraints +name: Abbreviated old naming fixture with no declared constraints tier: row_bearing origin: synthetic thresholdEligible: false diff --git a/packages/context/test/fixtures/relationship-benchmarks/abbreviated_legacy_no_declared_constraints/snapshot.json b/packages/context/test/fixtures/relationship-benchmarks/abbreviated_old_no_declared_constraints/snapshot.json similarity index 98% rename from packages/context/test/fixtures/relationship-benchmarks/abbreviated_legacy_no_declared_constraints/snapshot.json rename to packages/context/test/fixtures/relationship-benchmarks/abbreviated_old_no_declared_constraints/snapshot.json index ac3840e2..b4cb7a92 100644 --- a/packages/context/test/fixtures/relationship-benchmarks/abbreviated_legacy_no_declared_constraints/snapshot.json +++ b/packages/context/test/fixtures/relationship-benchmarks/abbreviated_old_no_declared_constraints/snapshot.json @@ -1,5 +1,5 @@ { - "connectionId": "abbreviated_legacy_no_declared_constraints", + "connectionId": "abbreviated_old_no_declared_constraints", "driver": "sqlite", "extractedAt": "2026-05-07T00:00:00.000Z", "scope": {}, diff --git a/python/ktx-sl/semantic_layer/loader.py b/python/ktx-sl/semantic_layer/loader.py index c6956c19..02b5b2c4 100644 --- a/python/ktx-sl/semantic_layer/loader.py +++ b/python/ktx-sl/semantic_layer/loader.py @@ -59,9 +59,7 @@ class SourceLoader: f"Duplicate source name '{name}' in manifest shard {path}" ) sources[name] = project_manifest_entry(name, entry) - description_sources[name] = _description_sources( - entry.descriptions, entry.description, entry.db_description - ) + description_sources[name] = _description_sources(entry.descriptions) # 2. Load files outside _schema/ for path in sorted(self.sources_dir.rglob("*.yaml")): @@ -138,11 +136,6 @@ class SourceLoader: source = deepcopy(base) description_sources = dict(base_description_sources or {}) - # Overlay description semantics match the server: `description` writes the - # `user` source key, and `descriptions` merges keyed sources before a single - # visible description is resolved from the full map. - if overlay.get("description"): - description_sources["user"] = overlay["description"] if overlay.get("descriptions"): description_sources.update( { @@ -151,7 +144,7 @@ class SourceLoader: if text } ) - if overlay.get("description") or overlay.get("descriptions"): + if overlay.get("descriptions"): source.description = _resolve_description( description_sources or None, ) diff --git a/python/ktx-sl/semantic_layer/manifest.py b/python/ktx-sl/semantic_layer/manifest.py index 3023cf58..432019e8 100644 --- a/python/ktx-sl/semantic_layer/manifest.py +++ b/python/ktx-sl/semantic_layer/manifest.py @@ -76,31 +76,17 @@ def map_column_type(db_type: str) -> str: _DEFAULT_PRIORITY = ["user", "ai", "dbt", "db"] -def _description_sources( - descriptions: dict[str, str] | None, - description: str | None = None, - db_description: str | None = None, -) -> dict[str, str] | None: +def _description_sources(descriptions: dict[str, str] | None) -> dict[str, str] | None: """Normalize multi-source descriptions to a keyed map.""" if descriptions: result = {source: text for source, text in descriptions.items() if text} if result: return result - - result: dict[str, str] = {} - if description: - result["ai"] = description - if db_description: - result["db"] = db_description - return result or None + return None -def _resolve_description( - descriptions: dict[str, str] | None, - description: str | None = None, - db_description: str | None = None, -) -> str | None: - """Resolve a single description from a multi-source map or legacy flat fields.""" +def _resolve_description(descriptions: dict[str, str] | None) -> str | None: + """Resolve a single description from a multi-source map.""" if descriptions: for source in _DEFAULT_PRIORITY: if text := descriptions.get(source): @@ -109,11 +95,6 @@ def _resolve_description( for text in descriptions.values(): if text: return text - # Legacy flat fields - if description: - return description - if db_description: - return db_description return None @@ -123,18 +104,13 @@ class ManifestColumn(BaseModel): pk: bool = False nullable: bool = True descriptions: dict[str, str] | None = None - # Legacy flat fields (backwards-compatible YAML parsing) - description: str | None = None - db_description: str | None = None constraints: dict | None = None enum_values: dict[str, list[str]] | None = None tests: SourceColumnTests | None = None @property def resolved_description(self) -> str | None: - return _resolve_description( - self.descriptions, self.description, self.db_description - ) + return _resolve_description(self.descriptions) class ManifestJoin(BaseModel): @@ -147,9 +123,6 @@ class ManifestJoin(BaseModel): class ManifestEntry(BaseModel): table: str descriptions: dict[str, str] | None = None - # Legacy flat fields (backwards-compatible YAML parsing) - description: str | None = None - db_description: str | None = None columns: list[ManifestColumn] joins: list[ManifestJoin] = [] default_time_dimension: DefaultTimeDimensionDbt | None = None @@ -158,9 +131,7 @@ class ManifestEntry(BaseModel): @property def resolved_description(self) -> str | None: - return _resolve_description( - self.descriptions, self.description, self.db_description - ) + return _resolve_description(self.descriptions) class Manifest(BaseModel): @@ -178,6 +149,8 @@ def validate_overlay(data: dict) -> list[str]: Returns a list of error messages (empty if valid). """ errors: list[str] = [] + if "description" in data: + errors.append("Overlay must use 'descriptions' for source descriptions") if "table" in data: errors.append("Overlay must not contain 'table' (owned by manifest)") if "sql" in data: @@ -185,6 +158,10 @@ def validate_overlay(data: dict) -> list[str]: "Overlay must not contain 'sql' (that makes it a standalone source)" ) for col in data.get("columns", []): + if "description" in col: + errors.append( + f"Overlay column '{col.get('name', '?')}' must use 'descriptions'" + ) if "type" in col and "expr" not in col: errors.append( f"Overlay column '{col.get('name', '?')}' specifies 'type' without 'expr' " diff --git a/python/ktx-sl/sources/b2b_saas/churn_risk.yaml b/python/ktx-sl/sources/b2b_saas/churn_risk.yaml index 2fae793e..602d263a 100644 --- a/python/ktx-sl/sources/b2b_saas/churn_risk.yaml +++ b/python/ktx-sl/sources/b2b_saas/churn_risk.yaml @@ -1,10 +1,11 @@ name: churn_risk -description: | - Per-account churn risk scoring for B2B SaaS customers. Combines signals from - subscriptions (cancellation history), support tickets (severity, SLA breaches), - product usage (adoption decline), contracts (renewal proximity), CSM activities - (engagement recency), and invoices (payment issues) into a weighted composite - risk_score (0-1) and risk_tier (High/Medium/Low). One row per customer account. +descriptions: + user: | + Per-account churn risk scoring for B2B SaaS customers. Combines signals from + subscriptions (cancellation history), support tickets (severity, SLA breaches), + product usage (adoption decline), contracts (renewal proximity), CSM activities + (engagement recency), and invoices (payment issues) into a weighted composite + risk_score (0-1) and risk_tier (High/Medium/Low). One row per customer account. sql: | WITH sub_signals AS ( SELECT diff --git a/python/ktx-sl/sources/ecommerce/churn_risk.yaml b/python/ktx-sl/sources/ecommerce/churn_risk.yaml index 32e919ed..7a009a59 100644 --- a/python/ktx-sl/sources/ecommerce/churn_risk.yaml +++ b/python/ktx-sl/sources/ecommerce/churn_risk.yaml @@ -1,7 +1,8 @@ name: churn_risk -description: | - Customer churn risk score combining tenure, - usage trends, and support burden. +descriptions: + user: | + Customer churn risk score combining tenure, + usage trends, and support burden. sql: | SELECT c.id AS customer_id, diff --git a/python/ktx-sl/tests/test_manifest.py b/python/ktx-sl/tests/test_manifest.py index e025c3da..1007fc89 100644 --- a/python/ktx-sl/tests/test_manifest.py +++ b/python/ktx-sl/tests/test_manifest.py @@ -95,7 +95,7 @@ class TestProjectManifestEntry: def orders_entry(self) -> ManifestEntry: return ManifestEntry( table="public.orders", - description="Customer orders", + descriptions={"user": "Customer orders"}, columns=[ ManifestColumn(name="id", type="integer", pk=True), ManifestColumn(name="customer_id", type="integer"), @@ -202,7 +202,7 @@ class TestValidateOverlay: def test_validate_overlay_valid(self): data = { "name": "orders", - "description": "Revenue-bearing orders", + "descriptions": {"user": "Revenue-bearing orders"}, "grain": ["id"], "measures": [{"name": "revenue", "expr": "sum(total)"}], "columns": [ @@ -259,7 +259,7 @@ def _manifest_tables() -> dict: "tables": { "orders": { "table": "public.orders", - "description": "Customer orders", + "descriptions": {"user": "Customer orders"}, "columns": [ {"name": "id", "type": "integer", "pk": True}, {"name": "customer_id", "type": "integer"}, @@ -278,7 +278,7 @@ def _manifest_tables() -> dict: }, "customers": { "table": "public.customers", - "description": "Customer accounts", + "descriptions": {"user": "Customer accounts"}, "columns": [ {"name": "id", "type": "integer", "pk": True}, {"name": "name", "type": "varchar"}, @@ -329,12 +329,12 @@ class TestTwoTierLoading: assert sources["regions"].table == "public.regions" assert sources["regions"].is_table_source - def test_overlay_descriptions_do_not_promote_base_description_to_user_source( + def test_overlay_descriptions_do_not_promote_base_map_to_user_source( self, tmp_path: Path ): standalone = { "name": "regions", - "description": "Standalone description", + "descriptions": {"ai": "Standalone description"}, "table": "public.regions", "grain": ["id"], "columns": [ @@ -376,7 +376,7 @@ class TestTwoTierLoading: overlay = { "name": "orders", - "description": "Revenue-bearing orders", + "descriptions": {"user": "Revenue-bearing orders"}, "grain": ["id"], "measures": [{"name": "revenue", "expr": "sum(total)"}], } @@ -394,11 +394,11 @@ class TestTwoTierLoading: assert len(orders.measures) == 1 assert orders.measures[0].name == "revenue" - def test_overlay_description_override(self, tmp_path: Path): + def test_overlay_description_map_override(self, tmp_path: Path): schema_dir = tmp_path / "_schema" _write_yaml(schema_dir / "public.yaml", _manifest_tables()) - overlay = {"name": "orders", "description": "Overridden description"} + overlay = {"name": "orders", "descriptions": {"user": "Overridden description"}} _write_yaml(tmp_path / "orders.yaml", overlay) _write_yaml(tmp_path / "customers.yaml", {"name": "customers"}) @@ -426,7 +426,7 @@ class TestTwoTierLoading: sources = loader.load_all() assert sources["orders"].description == "Customer orders" - def test_overlay_descriptions_map_overrides_lower_priority_db_description( + def test_overlay_descriptions_map_overrides_lower_priority_db_source( self, tmp_path: Path ): schema_dir = tmp_path / "_schema" diff --git a/scripts/build-evidence-fusion-adversarial-fixtures.mjs b/scripts/build-evidence-fusion-adversarial-fixtures.mjs index 6994d5c2..282a6477 100644 --- a/scripts/build-evidence-fusion-adversarial-fixtures.mjs +++ b/scripts/build-evidence-fusion-adversarial-fixtures.mjs @@ -129,10 +129,10 @@ function nonEnglishFixture() { }; } -function abbreviatedLegacyFixture() { +function abbreviatedOldNamingFixture() { return { - id: 'abbreviated_legacy_no_declared_constraints', - name: 'Abbreviated legacy naming fixture with no declared constraints', + id: 'abbreviated_old_no_declared_constraints', + name: 'Abbreviated old naming fixture with no declared constraints', tier: 'row_bearing', sql: [ 'CREATE TABLE cust (cust_id TEXT NOT NULL, nm TEXT NOT NULL, stat_cd TEXT NOT NULL);', @@ -480,7 +480,7 @@ function scaleFixture() { const fixtures = [ nonEnglishFixture(), - abbreviatedLegacyFixture(), + abbreviatedOldNamingFixture(), analyticalWarehouseFixture(), mixedCaseFixture(), polymorphicFixture(), diff --git a/scripts/check-boundaries.mjs b/scripts/check-boundaries.mjs index 53455abd..9f2953e7 100644 --- a/scripts/check-boundaries.mjs +++ b/scripts/check-boundaries.mjs @@ -46,15 +46,15 @@ const llmBoundaryPatterns = [ pattern: /\bembedMany\b/, }, { - label: 'legacy context LLM provider port', + label: 'context-owned LLM provider port', pattern: /\bLlmProviderPort\b/, }, { - label: 'legacy scan LLM provider port', + label: 'scan-owned LLM provider port', pattern: /\bKtxScanLlmPort\b/, }, { - label: 'legacy gateway LLM provider helper', + label: 'context-owned gateway LLM provider helper', pattern: /\bcreateGatewayLlmProvider\b/, }, ]; diff --git a/scripts/check-boundaries.test.mjs b/scripts/check-boundaries.test.mjs index db8afafe..9d5bf6f9 100644 --- a/scripts/check-boundaries.test.mjs +++ b/scripts/check-boundaries.test.mjs @@ -92,7 +92,7 @@ describe('scanFileContent', () => { ); }); - it('rejects context-owned LLM provider construction after @ktx/llm migration', () => { + it('rejects context-owned LLM provider construction outside @ktx/llm', () => { const violations = [ ...scanFileContent( 'packages/context/src/agent/local-llm-provider.ts', diff --git a/scripts/examples-docs.test.mjs b/scripts/examples-docs.test.mjs index 62a25cf7..79e26f74 100644 --- a/scripts/examples-docs.test.mjs +++ b/scripts/examples-docs.test.mjs @@ -128,7 +128,6 @@ describe('standalone example docs', () => { .join('|'), ), ); - assert.doesNotMatch(readme, /--historic-sql-min-calls/); }); it('lists every workspace package in the contributor docs', async () => { From d7147f9ca1349138c92b4286930a083c7561d777 Mon Sep 17 00:00:00 2001 From: Andrey Avtomonov Date: Wed, 13 May 2026 16:05:58 +0200 Subject: [PATCH 2/2] feat: rename project wiki directory (#66) * feat: rename project wiki directory * test: fix wiki skill ordering expectations * Show configured context sources in setup --- README.md | 6 +- .../content/docs/cli-reference/ktx-wiki.mdx | 65 ++++++- .../content/docs/concepts/context-as-code.mdx | 16 +- .../docs/concepts/the-context-layer.mdx | 14 +- .../docs/getting-started/introduction.mdx | 2 +- .../docs/getting-started/quickstart.mdx | 8 +- .../content/docs/guides/building-context.mdx | 8 +- .../content/docs/guides/serving-agents.mdx | 6 +- .../content/docs/guides/writing-context.mdx | 27 +-- .../docs/integrations/agent-clients.mdx | 4 +- .../docs/integrations/context-sources.mdx | 14 +- docs-site/lib/llm-docs.ts | 8 +- examples/local-warehouse/ktx.yaml | 2 +- .../{knowledge => wiki}/global/revenue.md | 0 .../assets/demo/orbit/links/provenance.json | 32 ++-- packages/cli/assets/demo/orbit/manifest.json | 2 +- .../demo/orbit/replay.memory-flow.v1.json | 82 ++++----- .../orbit/{knowledge => wiki}/global/.gitkeep | 0 .../global/customer-communication-policy.md | 0 .../global/new-hire-onboarding-policy.md | 0 .../global/orbit-activation-kpi-glossary.md | 0 ...orbit-activation-policy-change-jan-2026.md | 0 .../orbit-arr-contract-first-definition.md | 0 .../global/orbit-company-overview.md | 0 .../orbit-customer-health-risk-definition.md | 0 .../orbit-customer-stakeholder-needs.md | 0 .../global/orbit-customers-source.md | 0 .../global/orbit-dbt-exposures.md | 0 .../global/orbit-dbt-project-overview.md | 0 .../global/orbit-how-we-work.md | 0 .../global/orbit-known-product-gaps.md | 0 .../global/orbit-mart-account-activity.md | 0 .../global/orbit-mart-account-segments.md | 0 .../global/orbit-mart-arr-daily.md | 0 .../global/orbit-mart-nrr-quarterly.md | 0 .../global/orbit-mart-procurement-activity.md | 0 .../orbit-mart-retention-movement-breakout.md | 0 .../global/orbit-mart-revenue-daily.md | 0 .../orbit-metabase-sql-library-patterns.md | 0 ...orbit-nrr-discount-expiration-treatment.md | 0 .../orbit-plan-segment-normalization.md | 0 .../orbit-procurement-qualifying-actions.md | 0 .../global/orbit-product-design-principles.md | 0 .../global/orbit-product-review-checklist.md | 0 ...bit-revenue-gross-to-net-reconciliation.md | 0 .../global/sales-ops-cs-handoff-process.md | 0 packages/cli/scripts/build-demo-assets.mjs | 88 +++++----- packages/cli/src/command-schemas.ts | 13 ++ .../cli/src/commands/knowledge-commands.ts | 49 +++++- packages/cli/src/demo-assets.test.ts | 6 +- packages/cli/src/demo-assets.ts | 6 +- packages/cli/src/index.test.ts | 68 +++++++- packages/cli/src/ingest.test-utils.ts | 6 +- packages/cli/src/knowledge.test.ts | 162 ++++++++++++------ packages/cli/src/knowledge.ts | 59 ++++++- packages/cli/src/memory-flow-hud.tsx | 4 +- .../cli/src/memory-flow-interactive.test.ts | 4 +- packages/cli/src/memory-flow-tui.test.tsx | 12 +- packages/cli/src/setup-context.test.ts | 8 +- packages/cli/src/setup-context.ts | 2 +- packages/cli/src/setup-demo-tour.ts | 4 +- packages/cli/src/setup-sources.test.ts | 26 +++ packages/cli/src/setup-sources.ts | 27 ++- .../context/prompts/memory_agent_backfill.md | 2 +- .../memory_agent_bundle_ingest_reconcile.md | 2 +- .../memory_agent_bundle_ingest_work_unit.md | 2 +- .../prompts/memory_agent_external_ingest.md | 2 +- .../context/prompts/memory_agent_research.md | 2 +- .../context/skills/metricflow_ingest/SKILL.md | 4 +- .../SKILL.md | 6 +- .../src/ingest/action-identity.test.ts | 4 +- .../src/ingest/adapters/dbt/dbt.adapter.ts | 2 +- .../local-ingest-acceptance.test.ts | 2 +- .../adapters/historic-sql/post-processor.ts | 2 +- .../adapters/historic-sql/projection.test.ts | 20 +-- .../adapters/historic-sql/projection.ts | 2 +- .../src/ingest/ingest-bundle.runner.test.ts | 16 +- .../src/ingest/ingest-bundle.runner.ts | 6 +- .../src/ingest/ingest-runtime-assets.test.ts | 4 +- .../src/ingest/local-bundle-runtime.ts | 14 +- .../ingest/memory-flow/acceptance-fixtures.ts | 16 +- .../src/ingest/memory-flow/events.test.ts | 10 +- .../ingest/memory-flow/interaction.test.ts | 14 +- .../memory-flow/interactive-render.test.ts | 8 +- .../src/ingest/memory-flow/render.test.ts | 4 +- .../src/ingest/memory-flow/schema.test.ts | 8 +- .../src/ingest/memory-flow/view-model.test.ts | 20 +-- .../src/ingest/report-snapshot.test.ts | 8 +- .../ingest/sqlite-bundle-ingest-store.test.ts | 8 +- .../context/src/ingest/wiki-sl-ref-repair.ts | 2 +- packages/context/src/mcp/context-tools.ts | 20 +-- .../src/mcp/local-project-ports.test.ts | 4 +- packages/context/src/mcp/server.test.ts | 24 +-- .../context/src/memory/capture-signals.ts | 4 +- .../context/src/memory/local-memory.test.ts | 10 +- packages/context/src/memory/local-memory.ts | 6 +- .../src/memory/memory-agent.service.ts | 6 +- .../context/src/memory/memory-runs.test.ts | 4 +- .../src/memory/memory-runtime-assets.test.ts | 6 +- packages/context/src/project/config.test.ts | 2 +- packages/context/src/project/config.ts | 2 +- .../src/project/local-git-file-store.test.ts | 22 +-- packages/context/src/project/project.test.ts | 6 +- packages/context/src/project/project.ts | 2 +- .../skills/skills-registry.service.test.ts | 28 +-- .../src/skills/skills-registry.service.ts | 2 +- .../src/wiki/knowledge-wiki.service.test.ts | 22 +-- .../src/wiki/knowledge-wiki.service.ts | 28 +-- .../context/src/wiki/local-knowledge.test.ts | 35 +++- packages/context/src/wiki/local-knowledge.ts | 10 +- .../src/wiki/sqlite-knowledge-index.test.ts | 30 ++-- .../src/wiki/tools/wiki-remove.tool.ts | 2 +- .../src/wiki/tools/wiki-search.tool.test.ts | 4 +- .../context/src/wiki/tools/wiki-write.tool.ts | 2 +- scripts/package-artifacts.mjs | 8 +- scripts/package-artifacts.test.mjs | 4 +- 116 files changed, 839 insertions(+), 484 deletions(-) rename examples/local-warehouse/{knowledge => wiki}/global/revenue.md (100%) rename packages/cli/assets/demo/orbit/{knowledge => wiki}/global/.gitkeep (100%) rename packages/cli/assets/demo/orbit/{knowledge => wiki}/global/customer-communication-policy.md (100%) rename packages/cli/assets/demo/orbit/{knowledge => wiki}/global/new-hire-onboarding-policy.md (100%) rename packages/cli/assets/demo/orbit/{knowledge => wiki}/global/orbit-activation-kpi-glossary.md (100%) rename packages/cli/assets/demo/orbit/{knowledge => wiki}/global/orbit-activation-policy-change-jan-2026.md (100%) rename packages/cli/assets/demo/orbit/{knowledge => wiki}/global/orbit-arr-contract-first-definition.md (100%) rename packages/cli/assets/demo/orbit/{knowledge => wiki}/global/orbit-company-overview.md (100%) rename packages/cli/assets/demo/orbit/{knowledge => wiki}/global/orbit-customer-health-risk-definition.md (100%) rename packages/cli/assets/demo/orbit/{knowledge => wiki}/global/orbit-customer-stakeholder-needs.md (100%) rename packages/cli/assets/demo/orbit/{knowledge => wiki}/global/orbit-customers-source.md (100%) rename packages/cli/assets/demo/orbit/{knowledge => wiki}/global/orbit-dbt-exposures.md (100%) rename packages/cli/assets/demo/orbit/{knowledge => wiki}/global/orbit-dbt-project-overview.md (100%) rename packages/cli/assets/demo/orbit/{knowledge => wiki}/global/orbit-how-we-work.md (100%) rename packages/cli/assets/demo/orbit/{knowledge => wiki}/global/orbit-known-product-gaps.md (100%) rename packages/cli/assets/demo/orbit/{knowledge => wiki}/global/orbit-mart-account-activity.md (100%) rename packages/cli/assets/demo/orbit/{knowledge => wiki}/global/orbit-mart-account-segments.md (100%) rename packages/cli/assets/demo/orbit/{knowledge => wiki}/global/orbit-mart-arr-daily.md (100%) rename packages/cli/assets/demo/orbit/{knowledge => wiki}/global/orbit-mart-nrr-quarterly.md (100%) rename packages/cli/assets/demo/orbit/{knowledge => wiki}/global/orbit-mart-procurement-activity.md (100%) rename packages/cli/assets/demo/orbit/{knowledge => wiki}/global/orbit-mart-retention-movement-breakout.md (100%) rename packages/cli/assets/demo/orbit/{knowledge => wiki}/global/orbit-mart-revenue-daily.md (100%) rename packages/cli/assets/demo/orbit/{knowledge => wiki}/global/orbit-metabase-sql-library-patterns.md (100%) rename packages/cli/assets/demo/orbit/{knowledge => wiki}/global/orbit-nrr-discount-expiration-treatment.md (100%) rename packages/cli/assets/demo/orbit/{knowledge => wiki}/global/orbit-plan-segment-normalization.md (100%) rename packages/cli/assets/demo/orbit/{knowledge => wiki}/global/orbit-procurement-qualifying-actions.md (100%) rename packages/cli/assets/demo/orbit/{knowledge => wiki}/global/orbit-product-design-principles.md (100%) rename packages/cli/assets/demo/orbit/{knowledge => wiki}/global/orbit-product-review-checklist.md (100%) rename packages/cli/assets/demo/orbit/{knowledge => wiki}/global/orbit-revenue-gross-to-net-reconciliation.md (100%) rename packages/cli/assets/demo/orbit/{knowledge => wiki}/global/sales-ops-cs-handoff-process.md (100%) rename packages/context/skills/{knowledge_capture => wiki_capture}/SKILL.md (97%) diff --git a/README.md b/README.md index b52a31f6..563525e5 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ reviewable project files that agents can use while planning, querying, and updating analytics work. A KTX project is a directory of plain files — YAML semantic sources, Markdown -knowledge pages, and SQLite state — that you commit to git and review in PRs, +wiki pages, and SQLite state — that you commit to git and review in PRs, just like dbt models. ## Who KTX is for @@ -105,7 +105,7 @@ my-project/ │ ├── orders.yaml # Semantic source definitions │ ├── customers.yaml │ └── order_items.yaml -├── knowledge/ +├── wiki/ │ ├── global/ │ │ ├── revenue.md # Business definitions and rules │ │ └── segment-classification.md @@ -118,7 +118,7 @@ my-project/ └── db.sqlite # Local state (git-ignored) ``` -Semantic sources and knowledge pages are committed to git. The `.ktx/` directory +Semantic sources and wiki pages are committed to git. The `.ktx/` directory holds ephemeral state and is git-ignored — delete it and KTX rebuilds on the next run. diff --git a/docs-site/content/docs/cli-reference/ktx-wiki.mdx b/docs-site/content/docs/cli-reference/ktx-wiki.mdx index 8e27b5ff..1d57a93f 100644 --- a/docs-site/content/docs/cli-reference/ktx-wiki.mdx +++ b/docs-site/content/docs/cli-reference/ktx-wiki.mdx @@ -1,9 +1,9 @@ --- title: "ktx wiki" -description: "List or search knowledge pages." +description: "List, read, search, or write wiki pages." --- -Manage knowledge pages in your KTX project. Knowledge pages are Markdown documents that capture business definitions, rules, and gotchas. Agents search them for context when answering questions about your data. +Manage wiki pages in your KTX project. Wiki pages are Markdown documents that capture business definitions, rules, and gotchas. Agents search them for context when answering questions about your data. ## Command signature @@ -16,7 +16,9 @@ ktx wiki [options] | Subcommand | Description | |-----------|-------------| | `list` | List local wiki pages | +| `read ` | Read one local wiki page | | `search ` | Search local wiki pages | +| `write ` | Write one local wiki page | ## Options @@ -27,6 +29,13 @@ ktx wiki [options] | `--json` | Print JSON output | `false` | | `--user-id ` | Local user id | `local` | +### `wiki read` + +| Flag | Description | Default | +|------|-------------|---------| +| `--json` | Print JSON output | `false` | +| `--user-id ` | Local user id | `local` | + ### `wiki search` | Flag | Description | Default | @@ -35,6 +44,18 @@ ktx wiki [options] | `--user-id ` | Local user id | `local` | | `--limit ` | Maximum search results | — | +### `wiki write` + +| Flag | Description | Default | +|------|-------------|---------| +| `--user-id ` | Local user id | `local` | +| `--scope ` | Scope: `global` or `user` | `global` | +| `--summary ` | Wiki page summary (required) | — | +| `--content ` | Wiki page content (required) | — | +| `--tag ` | Wiki tag; repeatable | — | +| `--ref ` | Wiki ref; repeatable | — | +| `--sl-ref ` | Semantic-layer ref; repeatable | — | + ## Examples ```bash @@ -44,16 +65,48 @@ ktx wiki list # List all wiki pages as JSON ktx wiki list --json +# Read a specific wiki page +ktx wiki read revenue-definitions + +# Read a specific wiki page as JSON +ktx wiki read revenue-definitions --json + # Search wiki pages ktx wiki search "monthly recurring revenue" # Search wiki pages as JSON ktx wiki search "monthly recurring revenue" --json --limit 10 + +# Write a global wiki page +ktx wiki write revenue-definitions \ + --summary "Canonical revenue metric definitions" \ + --content "## MRR\nMonthly Recurring Revenue is calculated as..." + +# Write a user-scoped wiki page +ktx wiki write my-notes \ + --scope user \ + --summary "Personal analysis notes" \ + --content "Things to check when revenue numbers look off..." + +# Write a page with tags and references +ktx wiki write churn-rules \ + --summary "Churn calculation business rules" \ + --content "A customer is considered churned when..." \ + --tag finance \ + --tag retention \ + --sl-ref customers \ + --sl-ref subscriptions + +# Write a page with external references +ktx wiki write data-freshness \ + --summary "Data pipeline SLAs and freshness guarantees" \ + --content "The orders table refreshes every 15 minutes..." \ + --ref "https://wiki.example.com/data-pipelines" ``` ## Output -Wiki commands print local knowledge pages and search results. +Wiki commands print local wiki pages and search results. Agents should search first, then read the most relevant page by key. ```json { @@ -74,5 +127,7 @@ Wiki commands print local knowledge pages and search results. | Error | Cause | Recovery | |-------|-------|----------| -| Search returns no results | The query terms do not match summaries, tags, or content | Retry with business synonyms or run ingest to capture more context | -| A page is missing | The page has not been created by ingest or memory capture yet | Run ingest, then search again with `ktx wiki search` | +| Search returns no results | The query terms do not match summaries, tags, or content | Retry with business synonyms, then create a page if the knowledge is missing | +| Read fails for a key | The page key is wrong or scoped to a different user | Run `ktx wiki list` or search again to get the exact key | +| Write fails due to missing fields | `--summary` or `--content` was omitted | Pass both fields, and keep the summary short enough for search results | +| Agent writes duplicate pages | It did not search existing pages first | Always run `ktx wiki search` before `ktx wiki write` | diff --git a/docs-site/content/docs/concepts/context-as-code.mdx b/docs-site/content/docs/concepts/context-as-code.mdx index 3c43082e..51141b85 100644 --- a/docs-site/content/docs/concepts/context-as-code.mdx +++ b/docs-site/content/docs/concepts/context-as-code.mdx @@ -7,9 +7,9 @@ description: Treat analytics context like code — version it, review it, merge dbt proved that analytics transformations belong in version control. Before dbt, SQL lived in BI tools, scheduling systems, and spreadsheets — scattered, unreviewed, impossible to audit. "Analytics as code" changed that: put your models in git, review them in PRs, deploy them by merging. -KTX applies the same principle to analytics context. Metric definitions, business rules, join relationships, knowledge pages — these are artifacts that determine whether an agent produces correct results. They change over time. They need review. They need history. They need to be treated like code. +KTX applies the same principle to analytics context. Metric definitions, business rules, join relationships, wiki pages — these are artifacts that determine whether an agent produces correct results. They change over time. They need review. They need history. They need to be treated like code. -A KTX project is a git repository. Semantic sources are YAML files. Knowledge pages are Markdown files. Changes are commits. Updates are pull requests. Deployment is a merge. The entire lifecycle of your analytics context follows the same workflow your team already uses for dbt models, application code, and infrastructure. +A KTX project is a git repository. Semantic sources are YAML files. Wiki pages are Markdown files. Changes are commits. Updates are pull requests. Deployment is a merge. The entire lifecycle of your analytics context follows the same workflow your team already uses for dbt models, application code, and infrastructure. ## Auto-ingestion @@ -19,9 +19,9 @@ An ingestion run works like this: 1. **Adapters extract metadata.** Each configured source — dbt, LookML, Metabase, MetricFlow, Notion, or your live database — provides structured metadata about models, metrics, dimensions, questions, and documentation. -2. **The LLM agent reconciles.** KTX doesn't blindly overwrite existing context. An LLM agent compares incoming metadata against your current semantic sources and knowledge pages. It decides what to create, what to update, and what to leave alone. If your dbt project added a new model, the agent writes a new semantic source. If a Metabase question references a metric you've already defined, the agent skips the duplicate. +2. **The LLM agent reconciles.** KTX doesn't blindly overwrite existing context. An LLM agent compares incoming metadata against your current semantic sources and wiki pages. It decides what to create, what to update, and what to leave alone. If your dbt project added a new model, the agent writes a new semantic source. If a Metabase question references a metric you've already defined, the agent skips the duplicate. -3. **Files are written.** New and updated YAML sources and Markdown knowledge pages are written to the project directory. Every decision is recorded in the session transcript. +3. **Files are written.** New and updated YAML sources and Markdown wiki pages are written to the project directory. Every decision is recorded in the session transcript. This reconciliation step is what separates auto-ingestion from a simple sync. A naive import would overwrite your hand-tuned metric definitions every time dbt's manifest changes. KTX's agent-driven approach merges intelligently: it respects your edits, fills gaps, and flags conflicts for human review. @@ -43,7 +43,7 @@ dbt / Looker / Metabase / Notion | | + 3 new sources | ~ 2 updated joins - | + 1 knowledge page + | + 1 wiki page v open PR | @@ -57,7 +57,7 @@ dbt / Looker / Metabase / Notion agents see updated context ``` -A typical branch shows a semantic diff: "this ingest added 3 new sources from dbt, updated 2 join definitions based on schema changes, and created 1 knowledge page from a Notion doc." Analytics engineers review the diff, verify that the new sources look correct, and merge. +A typical branch shows a semantic diff: "this ingest added 3 new sources from dbt, updated 2 join definitions based on schema changes, and created 1 wiki page from a Notion doc." Analytics engineers review the diff, verify that the new sources look correct, and merge. Teams usually run this on demand while setting up a source, then schedule it once the source is stable. A cron job or CI schedule can run `ktx ingest run --connection-id --adapter --no-input` overnight on an ingest branch so the latest dbt manifests, BI metadata, and documentation updates are ready for review each morning. @@ -69,9 +69,9 @@ This workflow gives you the same review guarantees you have for dbt models. No s Context improves over time through two feedback channels. -**Analyst corrections.** When an analytics engineer spots something wrong — a measure formula that doesn't match the business definition, a join that should be `many_to_one` instead of `one_to_many`, a knowledge page that's out of date — they edit the YAML or Markdown directly and commit. These corrections become part of the project's git history, and the next ingestion run respects them. If you manually fix a measure definition, KTX won't overwrite it on the next ingest. +**Analyst corrections.** When an analytics engineer spots something wrong — a measure formula that doesn't match the business definition, a join that should be `many_to_one` instead of `one_to_many`, a wiki page that's out of date — they edit the YAML or Markdown directly and commit. These corrections become part of the project's git history, and the next ingestion run respects them. If you manually fix a measure definition, KTX won't overwrite it on the next ingest. -**Agent feedback.** When an agent queries the semantic layer and gets unexpected results — a query that returns no rows because of a bad filter, a join path that produces duplicated results — it can flag the issue. These signals feed back into the context: knowledge pages can note known data quality issues, and source definitions can be tightened with better filters, join paths, or grain declarations. +**Agent feedback.** When an agent queries the semantic layer and gets unexpected results — a query that returns no rows because of a bad filter, a join path that produces duplicated results — it can flag the issue. These signals feed back into the context: wiki pages can note known data quality issues, and source definitions can be tightened with better filters, join paths, or grain declarations. Each of these channels makes the next ingestion cycle better. Analyst corrections teach the system what your team considers authoritative. Agent feedback surfaces gaps in coverage. Context is not a static artifact — it's a living system that converges toward accuracy with every iteration. diff --git a/docs-site/content/docs/concepts/the-context-layer.mdx b/docs-site/content/docs/concepts/the-context-layer.mdx index 70480f48..d9021a8e 100644 --- a/docs-site/content/docs/concepts/the-context-layer.mdx +++ b/docs-site/content/docs/concepts/the-context-layer.mdx @@ -30,7 +30,7 @@ A context layer is the infrastructure that gives agents the business knowledge t KTX organizes context into four pillars: - Semantic sources -- Knowledge pages +- Wiki pages - Scan artifacts - Provenance @@ -67,7 +67,7 @@ measures: expr: count(id) ``` -**Knowledge pages** are Markdown documents that capture business definitions, rules, and operating context — the kind of context that doesn't fit in a schema definition. Pages have structured frontmatter (summary, tags, semantic layer references) and free-form content. Agents search them when they need to understand why a metric works a certain way, not just how to compute it. +**Wiki pages** are Markdown documents that capture business definitions, rules, and operating context — the kind of context that doesn't fit in a schema definition. Pages have structured frontmatter (summary, tags, semantic layer references) and free-form content. Agents search them when they need to understand why a metric works a certain way, not just how to compute it. ```markdown --- @@ -97,13 +97,13 @@ Together, these four pillars give agents enough context to produce analytics art ## How KTX compares -KTX is a context layer with an agent-native semantic layer at its core. MetricFlow, Cube, and Malloy model metrics, dimensions, joins, and generated SQL. KTX covers that semantic-layer work, then adds the context agents need to use and maintain it: knowledge pages, schema scans, provenance, ingestion, validation, and agent-facing CLI commands. +KTX is a context layer with an agent-native semantic layer at its core. MetricFlow, Cube, and Malloy model metrics, dimensions, joins, and generated SQL. KTX covers that semantic-layer work, then adds the context agents need to use and maintain it: wiki pages, schema scans, provenance, ingestion, validation, and agent-facing CLI commands. The workflow is the difference. Traditional semantic layers are powerful, but they are usually built and maintained through manual modeling work, product-specific runtimes, or language-specific workflows. They are not agent-native by default, which makes them harder for agents to inspect, edit, validate, and review in a tight loop. KTX is designed for agents that need to read context, change semantic files, inspect generated SQL, and leave a reviewable git diff. | | KTX semantic layer | MetricFlow | Cube | Malloy | |---|---|---|---|---| -| **Model surface** | Plain YAML sources plus Markdown knowledge pages | YAML semantic models and metrics in a dbt project | YAML or JavaScript cubes, views, access policies, and pre-aggregations | `.malloy` models, query pipelines, notebooks, and annotations | +| **Model surface** | Plain YAML sources plus Markdown wiki pages | YAML semantic models and metrics in a dbt project | YAML or JavaScript cubes, views, access policies, and pre-aggregations | `.malloy` models, query pipelines, notebooks, and annotations | | **What it models** | Sources, columns, measures, segments, joins, grain, filters, default time dimensions, and context references | Semantic models, entities, dimensions, measures, metrics, time grains, and metric types | Cubes, views, measures, dimensions, segments, joins, hierarchies, policies, and rollups | Sources, joins, dimensions, measures, calculations, nested results, and query pipelines | | **Agent edit loop** | First-class. Agents can patch small files, save imperfect drafts, run validation, query through the CLI, inspect SQL, and refine in the same workflow | Possible, but the interface is a dbt/metric workflow rather than an agent context workflow | Possible through code-first models and platform APIs, but changes are tied to runtime deployment and governance concerns | Possible, but agents must operate in Malloy's language and compiler model | | **Fan-out safety** | Explicit `grain` plus relationship metadata. KTX detects `one_to_many` fan-out, identifies chasm traps, pre-aggregates independent fact measures into CTEs, and rejects unsafe filters | Dataflow query planning for metric requests, multi-hop joins, metric time, and metric types | Runtime planner, modeled joins, primary keys, views, multi-fact views, and pre-aggregations | Symmetric aggregates and path-based aggregation in the language | @@ -111,7 +111,7 @@ The workflow is the difference. Traditional semantic layers are powerful, but th | **Context around semantics** | Built in: wiki pages, scan artifacts, relationship inference, ingest transcripts, replay, and agent-facing CLI commands | Primarily metric and dbt project context | Descriptions and `meta.ai_context` inside the semantic model, plus platform agent features | Annotations/tags can carry metadata; surrounding context depends on the application | | **Best fit** | Agents maintaining analytics code, metrics, joins, SQL, docs, and semantic definitions | Teams standardizing metrics inside dbt workflows | Production semantic APIs, BI integrations, access control, caching, and concurrency | Expressive modeling and exploratory analysis above SQL | -If you do not have a semantic layer, KTX can build an agent-native one from your database schema and enrich it with generated descriptions and knowledge pages. If you already use MetricFlow or LookML, KTX ingests from those tools and merges their context into KTX's files. You can keep your existing BI or metric-serving system while using KTX as the semantic and contextual surface agents work against. +If you do not have a semantic layer, KTX can build an agent-native one from your database schema and enrich it with generated descriptions and wiki pages. If you already use MetricFlow or LookML, KTX ingests from those tools and merges their context into KTX's files. You can keep your existing BI or metric-serving system while using KTX as the semantic and contextual surface agents work against. ## The plain-files philosophy @@ -125,7 +125,7 @@ my-project/ │ ├── orders.yaml # Semantic source definitions │ ├── customers.yaml │ └── order_items.yaml -├── knowledge/ +├── wiki/ │ ├── global/ │ │ ├── revenue.md # Business definitions and rules │ │ └── segment-classification.md @@ -140,7 +140,7 @@ my-project/ └── cache/ # Runtime cache (git-ignored) ``` -Semantic sources and knowledge pages are committed to git. The SQLite database holds ephemeral state — scan results, embedding indexes, session logs — and is git-ignored. If you delete it, KTX rebuilds it on the next run. +Semantic sources and wiki pages are committed to git. The SQLite database holds ephemeral state — scan results, embedding indexes, session logs — and is git-ignored. If you delete it, KTX rebuilds it on the next run. This means your analytics context travels with your code. You can fork it, branch it, review it in a PR, and merge it with the same tools you use for dbt models. There's no sync problem between a remote server and your local state. There's no migration to run. The files are the source of truth. diff --git a/docs-site/content/docs/getting-started/introduction.mdx b/docs-site/content/docs/getting-started/introduction.mdx index a9d98d3e..70ca9a84 100644 --- a/docs-site/content/docs/getting-started/introduction.mdx +++ b/docs-site/content/docs/getting-started/introduction.mdx @@ -88,5 +88,5 @@ Works with PostgreSQL, Snowflake, BigQuery, ClickHouse, MySQL, and SQL Server. | Set up a new KTX project | [Quickstart](/docs/getting-started/quickstart) | | Explain what problem KTX solves | [The Context Layer](/docs/concepts/the-context-layer) | | Scan a database and ingest metadata | [Building Context](/docs/guides/building-context) | -| Edit semantic sources or knowledge pages | [Writing Context](/docs/guides/writing-context) | +| Edit semantic sources or wiki pages | [Writing Context](/docs/guides/writing-context) | | Look up exact command flags | [CLI Reference](/docs/cli-reference/ktx-setup) | diff --git a/docs-site/content/docs/getting-started/quickstart.mdx b/docs-site/content/docs/getting-started/quickstart.mdx index 59a512cb..7aba00fd 100644 --- a/docs-site/content/docs/getting-started/quickstart.mdx +++ b/docs-site/content/docs/getting-started/quickstart.mdx @@ -146,7 +146,7 @@ This is where KTX does the heavy lifting. It runs an enriched scan of your datab │ ○ Leave context unbuilt and exit setup ``` -The build scans each primary source with LLM enrichment, detects table relationships, and runs ingestion agents that reconcile metadata from your context sources into semantic-layer YAML files and knowledge pages. +The build scans each primary source with LLM enrichment, detects table relationships, and runs ingestion agents that reconcile metadata from your context sources into semantic-layer YAML files and wiki pages. For a small database (under 50 tables), this takes a few minutes. Larger warehouses can take longer. You can press d to detach and let it run in the background: @@ -209,8 +209,8 @@ KTX writes project state as plain files so agents can inspect and edit changes i | `ktx.yaml` | `ktx setup` | Main project configuration: connections, LLM settings, embeddings, and context sources | | `.ktx/secrets/*` | `ktx setup` when file-backed secrets are selected | Local secret files referenced from `ktx.yaml`; do not commit these | | `semantic-layer//*.yaml` | context build, ingestion, or direct file edits | Semantic source definitions agents use for SQL generation | -| `knowledge/global/*.md` | ingestion, memory capture, or direct file edits | Shared business context and metric definitions | -| `knowledge/user//*.md` | memory capture or direct file edits | User-scoped notes for one agent/user context | +| `wiki/global/*.md` | ingestion, memory capture, `ktx wiki write --scope global`, or direct file edits | Shared business context and metric definitions | +| `wiki/user//*.md` | memory capture, `ktx wiki write --scope user`, or direct file edits | User-scoped notes for one agent/user context | | `.claude/skills/ktx/SKILL.md`, `.agents/skills/ktx/SKILL.md` | CLI-mode agent integration setup | Agent instructions for calling public `ktx` commands | ## Verify it worked @@ -247,6 +247,6 @@ Agent integration ready: yes (claude-code:project) ## Next steps - **Build more context** — learn about [scanning](/docs/guides/building-context), relationship detection, and ingestion workflows in the Building Context guide. -- **Refine your semantic layer** — the [Writing Context](/docs/guides/writing-context) guide covers source YAML, measures, joins, and knowledge pages. +- **Refine your semantic layer** — the [Writing Context](/docs/guides/writing-context) guide covers source YAML, measures, joins, and wiki pages. - **Understand the architecture** — read [The Context Layer](/docs/concepts/the-context-layer) to learn why a context layer is more than a semantic layer. - **Connect more agents** — see the [Agent Clients](/docs/integrations/agent-clients) integration page for per-tool setup details. diff --git a/docs-site/content/docs/guides/building-context.mdx b/docs-site/content/docs/guides/building-context.mdx index 25d873d9..c3821a52 100644 --- a/docs-site/content/docs/guides/building-context.mdx +++ b/docs-site/content/docs/guides/building-context.mdx @@ -53,7 +53,7 @@ Relationship scans run with `ktx scan --mode relationships`. Thi ## Ingestion -Ingestion pulls semantic context from your existing analytics tools — dbt projects, Looker models, Metabase questions, and more — and writes it into your KTX project as semantic sources and knowledge pages. +Ingestion pulls semantic context from your existing analytics tools — dbt projects, Looker models, Metabase questions, and more — and writes it into your KTX project as semantic sources and wiki pages. ### How it works @@ -61,7 +61,7 @@ Each ingest run follows this flow: 1. An **adapter** extracts metadata from your tool (dbt manifest, LookML files, Metabase API, etc.) 2. An **LLM agent** reconciles the extracted metadata with your existing context — it merges intelligently rather than overwriting -3. **Semantic sources** (YAML) and **knowledge pages** (Markdown) are written to your project directory +3. **Semantic sources** (YAML) and **wiki pages** (Markdown) are written to your project directory ### Running an ingest @@ -113,7 +113,7 @@ See [Context Sources](/docs/integrations/context-sources) for adapter-specific s ### What gets generated -A typical dbt ingest produces semantic sources and knowledge pages in your project: +A typical dbt ingest produces semantic sources and wiki pages in your project: **Semantic source** (`semantic-layer/my-postgres/orders.yaml`): @@ -149,7 +149,7 @@ joins: relationship: many_to_one ``` -**Knowledge page** (`knowledge/global/order-status-definitions.md`): +**Wiki page** (`wiki/global/order-status-definitions.md`): ```markdown --- diff --git a/docs-site/content/docs/guides/serving-agents.mdx b/docs-site/content/docs/guides/serving-agents.mdx index 0de6934e..4a93ae43 100644 --- a/docs-site/content/docs/guides/serving-agents.mdx +++ b/docs-site/content/docs/guides/serving-agents.mdx @@ -36,10 +36,10 @@ ktx sl query --json \ --max-rows 100 ``` -**Knowledge:** +**Wiki:** ```bash -# Search knowledge pages +# Search wiki pages ktx wiki search "revenue recognition" --json --limit 10 ``` @@ -56,4 +56,4 @@ configuration. For manual setup or per-tool details, see the [Agent Clients](/docs/integrations/agent-clients) integration page. After configuration, the agent can immediately call KTX commands to list -sources, search knowledge, and query your semantic layer. +sources, search wiki pages, and query your semantic layer. diff --git a/docs-site/content/docs/guides/writing-context.mdx b/docs-site/content/docs/guides/writing-context.mdx index 9e08fcc7..b6ca3597 100644 --- a/docs-site/content/docs/guides/writing-context.mdx +++ b/docs-site/content/docs/guides/writing-context.mdx @@ -1,9 +1,9 @@ --- title: Writing Context -description: Write and refine semantic sources and knowledge pages. +description: Write and refine semantic sources and wiki pages. --- -After building context through scanning and ingestion, you'll want to refine it — edit semantic sources to match your business logic, add knowledge pages that capture tribal knowledge, and query your data through the semantic layer to verify everything works. +After building context through scanning and ingestion, you'll want to refine it — edit semantic sources to match your business logic, add wiki pages that capture tribal knowledge, and query your data through the semantic layer to verify everything works. ## Agent workflow summary @@ -218,20 +218,20 @@ The query planner is grain-aware — it understands the cardinality of joins and If validation fails, fix the YAML before asking an agent to use the source. Common validation failures are missing columns, invalid join targets, and measure expressions that reference fields outside the source. -## Knowledge Pages +## Wiki Pages -Knowledge pages are Markdown files that capture business context — definitions, rules, gotchas, and anything an agent needs to understand beyond what the schema tells it. +Wiki pages are Markdown files that capture business context — definitions, rules, gotchas, and anything an agent needs to understand beyond what the schema tells it. ### What they are -When an agent asks "what counts as an active user?" or "why do revenue numbers differ between the dashboard and the SQL query?", the answer isn't in the schema. It's tribal knowledge that lives in Slack threads, Notion pages, or someone's head. Knowledge pages make that context searchable and available to agents. +When an agent asks "what counts as an active user?" or "why do revenue numbers differ between the dashboard and the SQL query?", the answer isn't in the schema. It's tribal knowledge that lives in Slack threads, Notion pages, or someone's head. Wiki pages make that context searchable and available to agents. ### Organization -Knowledge pages are organized by scope: +Wiki pages are organized by scope: ``` -knowledge/ +wiki/ ├── global/ # Cross-cutting definitions │ ├── order-status-definitions.md │ ├── revenue-recognition-rules.md @@ -247,10 +247,11 @@ knowledge/ ### Editing pages -Create and edit knowledge pages directly as Markdown files in the `knowledge/` -directory. Ingest and memory capture also create these pages automatically. +Create and edit wiki pages directly as Markdown files in the `wiki/` +directory, or with `ktx wiki write`. Ingest and memory capture also create +these pages automatically. -Knowledge page fields: +Wiki page fields: | Field | Required | Description | |-------|----------|-------------| @@ -279,7 +280,7 @@ Search uses both full-text matching and semantic similarity — it finds relevan ### Workflow: add searchable business context 1. Search first: `ktx wiki search "order status definitions"`. -2. If no page already covers the rule, create or edit a Markdown file under `knowledge/global/`. +2. If no page already covers the rule, create or edit a Markdown file under `wiki/global/`. 3. Include concise frontmatter; agents see the summary before loading full content. 4. Add `tags` values for the business area and `sl_refs` values for related semantic sources. 5. Search again with the user's likely wording to confirm the page is discoverable. @@ -290,6 +291,6 @@ Search uses both full-text matching and semantic similarity — it finds relevan |------------------|--------------|----------| | `ktx sl validate` reports a missing column | YAML references a column that is absent from the scanned table | Run a fresh scan or update the YAML to match the warehouse schema | | Query compilation double-counts a measure | Join relationship or grain is missing or wrong | Add `grain` and explicit `relationship` values, then validate and recompile | -| Agent cannot find a metric | Measure name or description does not match business terminology | Add a measure description and a knowledge page with common synonyms | -| Knowledge search misses a page | Summary and tags do not include likely user wording | Rewrite the summary and add relevant tags, then search again | +| Agent cannot find a metric | Measure name or description does not match business terminology | Add a measure description and a wiki page with common synonyms | +| Wiki search misses a page | Summary and tags do not include likely user wording | Rewrite the summary and add relevant tags, then search again | | Semantic-layer changes are hard to review | The YAML edit is too large or unfocused | Split the change into smaller source-file edits, then review the git diff | diff --git a/docs-site/content/docs/integrations/agent-clients.mdx b/docs-site/content/docs/integrations/agent-clients.mdx index 61538140..95786f52 100644 --- a/docs-site/content/docs/integrations/agent-clients.mdx +++ b/docs-site/content/docs/integrations/agent-clients.mdx @@ -124,7 +124,9 @@ All supported agent clients call the same KTX CLI commands: | Command | Description | |---------|-------------| | `ktx status --json` | Return project setup and context readiness | -| `ktx wiki search --json` | Search knowledge pages | +| `ktx wiki search --json` | Search wiki pages | +| `ktx wiki read --json` | Read a wiki page | +| `ktx wiki write ` | Write or update a wiki page | | `ktx sl list --json` | List semantic-layer sources | | `ktx sl search --json` | Search semantic-layer sources | | `ktx sl validate --connection-id ` | Validate semantic source definitions | diff --git a/docs-site/content/docs/integrations/context-sources.mdx b/docs-site/content/docs/integrations/context-sources.mdx index 904e3f95..5b85bff2 100644 --- a/docs-site/content/docs/integrations/context-sources.mdx +++ b/docs-site/content/docs/integrations/context-sources.mdx @@ -15,7 +15,7 @@ Agents should configure and ingest context sources in this order: 2. Store tokens as `env:NAME` or `file:/path/to/secret`. 3. Run `ktx ingest run --connection-id --adapter ` for one source or `ktx ingest run --connection-id --adapter `. 4. Check progress with `ktx ingest status --json`. -5. Review generated `semantic-layer/` YAML and `knowledge/` Markdown files in git. +5. Review generated `semantic-layer/` YAML and `wiki/` Markdown files in git. 6. Validate changed semantic sources with `ktx sl validate`. ## Shared source fields @@ -233,7 +233,7 @@ Generate an API key in Metabase: **Admin > Settings > Authentication > API Keys* ### What gets ingested - Semantic sources generated from SQL queries in questions -- Knowledge pages for dashboards (purpose, key metrics, relationships) +- Wiki pages for dashboards (purpose, key metrics, relationships) - Work units per dashboard and per question ### Warehouse mapping @@ -290,7 +290,7 @@ Generate API credentials in Looker: **Admin > Users > Edit > API Keys**. ### What gets ingested - Semantic sources from explore field definitions -- Knowledge pages for dashboards (purpose, audience, key metrics) +- Wiki pages for dashboards (purpose, audience, key metrics) - Triage signals for automated content classification - Work units per explore and per dashboard @@ -310,11 +310,11 @@ Find Looker connection names in **Admin > Database > Connections**. ## Notion -Ingests pages and databases from a Notion workspace as knowledge pages. Useful for capturing business definitions, data dictionaries, and team documentation that agents need for context. +Ingests pages and databases from a Notion workspace as wiki pages. Useful for capturing business definitions, data dictionaries, and team documentation that agents need for context. ### What it provides -- Knowledge pages synthesized from Notion content +- Wiki pages synthesized from Notion content - Page hierarchy and relationships - Database schemas (when Notion databases describe data sources) - Semantic clustering for organized ingestion @@ -364,7 +364,7 @@ Create an integration at [notion.so/my-integrations](https://www.notion.so/my-in ### What gets ingested -- Knowledge pages synthesized from Notion content (not raw copies) +- Wiki pages synthesized from Notion content (not raw copies) - Domain context extracted and organized by topic - Triage signals for classifying page relevance - Work units clustered by semantic similarity for efficient processing @@ -381,6 +381,6 @@ Create an integration at [notion.so/my-integrations](https://www.notion.so/my-in |------------------|--------------|----------| | Adapter cannot read source files | `source_dir`, `repo_url`, `branch`, or `path` is wrong | Verify the path locally or clone the repo manually with the same credentials | | Private repo/API authentication fails | Token env var or secret file is missing | Export the env var or update `auth_token_ref` to a readable file | -| Ingest creates duplicate context | Existing source names or knowledge pages do not match imported terminology | Review the diff, rename duplicates, and add knowledge pages with canonical names | +| Ingest creates duplicate context | Existing source names or wiki pages do not match imported terminology | Review the diff, rename duplicates, and add wiki pages with canonical names | | Notion ingest skips pages | Integration lacks access or root ids are missing | Share pages with the Notion integration and set `root_page_ids` or use `all_accessible` carefully | | Generated semantic sources fail validation | Tool metadata does not match the live warehouse schema | Map BI/source databases to primary warehouse connections and rerun validation | diff --git a/docs-site/lib/llm-docs.ts b/docs-site/lib/llm-docs.ts index 69aac698..cbf9ba9e 100644 --- a/docs-site/lib/llm-docs.ts +++ b/docs-site/lib/llm-docs.ts @@ -47,7 +47,7 @@ export function buildLlmsTxt() { > Agent-native context layer for analytics engineering and database agents. -KTX provides semantic-layer files, warehouse scans, knowledge pages, provenance, and agent-facing tools that help coding agents answer analytics questions without inventing metrics or joins. +KTX provides semantic-layer files, warehouse scans, wiki pages, provenance, and agent-facing tools that help coding agents answer analytics questions without inventing metrics or joins. ## Agent Entry Points @@ -60,7 +60,7 @@ ${link("/docs/ai-resources/agent-instructions", "Agent Instructions", "Suggested ${link("/docs/getting-started/introduction", "Introduction", "What KTX is and who it is for")} ${link("/docs/getting-started/quickstart", "Quickstart", "Set up KTX and build your first context")} -${link("/docs/guides/writing-context", "Writing Context", "Write semantic sources and knowledge pages")} +${link("/docs/guides/writing-context", "Writing Context", "Write semantic sources and wiki pages")} ## Machine-Readable Documentation @@ -68,13 +68,13 @@ ${link("/docs/guides/writing-context", "Writing Context", "Write semantic source - [Markdown access guide](${absoluteUrl("/docs/ai-resources/markdown-access.md")}): How to fetch llms.txt, llms-full.txt, and per-page Markdown - [Quickstart markdown](${absoluteUrl("/docs/getting-started/quickstart.md")}): Human setup walkthrough - [Semantic-layer CLI markdown](${absoluteUrl("/docs/cli-reference/ktx-sl.md")}): Semantic-layer commands and JSON output -- [Wiki CLI markdown](${absoluteUrl("/docs/cli-reference/ktx-wiki.md")}): Knowledge page commands and JSON output +- [Wiki CLI markdown](${absoluteUrl("/docs/cli-reference/ktx-wiki.md")}): Wiki page commands and JSON output ## CLI Reference ${link("/docs/cli-reference/ktx-setup", "ktx setup", "Interactive project setup")} ${link("/docs/cli-reference/ktx-sl", "ktx sl", "Semantic-layer commands")} -${link("/docs/cli-reference/ktx-wiki", "ktx wiki", "Knowledge page commands")} +${link("/docs/cli-reference/ktx-wiki", "ktx wiki", "Wiki page commands")} ${link("/docs/cli-reference/ktx-connection", "ktx connection", "Connection management commands")} ## Integrations diff --git a/examples/local-warehouse/ktx.yaml b/examples/local-warehouse/ktx.yaml index 7ca51365..00ccffbd 100644 --- a/examples/local-warehouse/ktx.yaml +++ b/examples/local-warehouse/ktx.yaml @@ -19,7 +19,7 @@ agent: max_iterations: 20 default_toolset: - sl_query - - knowledge_search + - wiki_search - sl_read_source memory: auto_commit: true diff --git a/examples/local-warehouse/knowledge/global/revenue.md b/examples/local-warehouse/wiki/global/revenue.md similarity index 100% rename from examples/local-warehouse/knowledge/global/revenue.md rename to examples/local-warehouse/wiki/global/revenue.md diff --git a/packages/cli/assets/demo/orbit/links/provenance.json b/packages/cli/assets/demo/orbit/links/provenance.json index 8b9e0b63..67dbb213 100644 --- a/packages/cli/assets/demo/orbit/links/provenance.json +++ b/packages/cli/assets/demo/orbit/links/provenance.json @@ -2,7 +2,7 @@ { "id": "link-001", "artifactKind": "wiki", - "artifactKey": "knowledge/global/arr-contract-first.md", + "artifactKey": "wiki/global/arr-contract-first.md", "sourceKind": "warehouse", "sourcePath": "contracts", "relationship": "describes", @@ -11,7 +11,7 @@ { "id": "link-002", "artifactKind": "wiki", - "artifactKey": "knowledge/global/arr-contract-first.md", + "artifactKey": "wiki/global/arr-contract-first.md", "sourceKind": "notion", "sourcePath": "raw-sources/notion/arr-and-contract-reporting-notes.md", "relationship": "derived_from", @@ -20,7 +20,7 @@ { "id": "link-003", "artifactKind": "wiki", - "artifactKey": "knowledge/global/revenue-gross-to-net.md", + "artifactKey": "wiki/global/revenue-gross-to-net.md", "sourceKind": "warehouse", "sourcePath": "invoices", "relationship": "describes", @@ -29,7 +29,7 @@ { "id": "link-004", "artifactKind": "wiki", - "artifactKey": "knowledge/global/revenue-gross-to-net.md", + "artifactKey": "wiki/global/revenue-gross-to-net.md", "sourceKind": "notion", "sourcePath": "raw-sources/notion/revenue-reporting-policy.md", "relationship": "derived_from", @@ -38,7 +38,7 @@ { "id": "link-005", "artifactKind": "wiki", - "artifactKey": "knowledge/global/discount-expiration.md", + "artifactKey": "wiki/global/discount-expiration.md", "sourceKind": "warehouse", "sourcePath": "arr_movements", "relationship": "describes", @@ -47,7 +47,7 @@ { "id": "link-006", "artifactKind": "wiki", - "artifactKey": "knowledge/global/nrr-retention.md", + "artifactKey": "wiki/global/nrr-retention.md", "sourceKind": "warehouse", "sourcePath": "arr_movements", "relationship": "describes", @@ -56,7 +56,7 @@ { "id": "link-007", "artifactKind": "wiki", - "artifactKey": "knowledge/global/nrr-retention.md", + "artifactKey": "wiki/global/nrr-retention.md", "sourceKind": "notion", "sourcePath": "raw-sources/notion/retention-and-nrr-definition-notes.md", "relationship": "derived_from", @@ -65,7 +65,7 @@ { "id": "link-008", "artifactKind": "wiki", - "artifactKey": "knowledge/global/nrr-retention.md", + "artifactKey": "wiki/global/nrr-retention.md", "sourceKind": "bi", "sourcePath": "raw-sources/bi/account_retention.view.lkml", "relationship": "derived_from", @@ -74,7 +74,7 @@ { "id": "link-009", "artifactKind": "wiki", - "artifactKey": "knowledge/global/segment-classification.md", + "artifactKey": "wiki/global/segment-classification.md", "sourceKind": "warehouse", "sourcePath": "plans", "relationship": "describes", @@ -83,7 +83,7 @@ { "id": "link-010", "artifactKind": "wiki", - "artifactKey": "knowledge/global/segment-classification.md", + "artifactKey": "wiki/global/segment-classification.md", "sourceKind": "notion", "sourcePath": "raw-sources/notion/sales-ops-segmentation-guide.md", "relationship": "derived_from", @@ -92,7 +92,7 @@ { "id": "link-011", "artifactKind": "wiki", - "artifactKey": "knowledge/global/activation-policy.md", + "artifactKey": "wiki/global/activation-policy.md", "sourceKind": "notion", "sourcePath": "raw-sources/notion/activation-policy-decision-record.md", "relationship": "derived_from", @@ -101,7 +101,7 @@ { "id": "link-012", "artifactKind": "wiki", - "artifactKey": "knowledge/global/procurement-workflows.md", + "artifactKey": "wiki/global/procurement-workflows.md", "sourceKind": "warehouse", "sourcePath": "purchase_requests", "relationship": "describes", @@ -110,7 +110,7 @@ { "id": "link-013", "artifactKind": "wiki", - "artifactKey": "knowledge/global/customer-health-scoring.md", + "artifactKey": "wiki/global/customer-health-scoring.md", "sourceKind": "notion", "sourcePath": "raw-sources/notion/customer-health-playbook.md", "relationship": "derived_from", @@ -119,7 +119,7 @@ { "id": "link-014", "artifactKind": "wiki", - "artifactKey": "knowledge/global/customer-health-scoring.md", + "artifactKey": "wiki/global/customer-health-scoring.md", "sourceKind": "warehouse", "sourcePath": "support_tickets", "relationship": "describes", @@ -128,7 +128,7 @@ { "id": "link-015", "artifactKind": "wiki", - "artifactKey": "knowledge/global/support-escalation.md", + "artifactKey": "wiki/global/support-escalation.md", "sourceKind": "notion", "sourcePath": "raw-sources/notion/support-escalation-runbook.md", "relationship": "derived_from", @@ -137,7 +137,7 @@ { "id": "link-016", "artifactKind": "wiki", - "artifactKey": "knowledge/global/internal-test-exclusion.md", + "artifactKey": "wiki/global/internal-test-exclusion.md", "sourceKind": "notion", "sourcePath": "raw-sources/notion/analyst-onboarding.md", "relationship": "derived_from", diff --git a/packages/cli/assets/demo/orbit/manifest.json b/packages/cli/assets/demo/orbit/manifest.json index 1fcb3bef..102c57aa 100644 --- a/packages/cli/assets/demo/orbit/manifest.json +++ b/packages/cli/assets/demo/orbit/manifest.json @@ -47,7 +47,7 @@ "sourceCount": 46 }, "knowledge": { - "path": "knowledge/global", + "path": "wiki/global", "pageCount": 28 }, "links": { diff --git a/packages/cli/assets/demo/orbit/replay.memory-flow.v1.json b/packages/cli/assets/demo/orbit/replay.memory-flow.v1.json index af4c1aa9..7bc4da18 100644 --- a/packages/cli/assets/demo/orbit/replay.memory-flow.v1.json +++ b/packages/cli/assets/demo/orbit/replay.memory-flow.v1.json @@ -71,7 +71,7 @@ "type": "work_unit_started", "unitKey": "revenue-and-contracts", "skills": [ - "knowledge_capture", + "wiki_capture", "sl_capture" ], "stepBudget": 40 @@ -81,21 +81,21 @@ "unitKey": "revenue-and-contracts", "target": "wiki", "action": "created", - "key": "knowledge/global/arr-contract-first.md" + "key": "wiki/global/arr-contract-first.md" }, { "type": "candidate_action", "unitKey": "revenue-and-contracts", "target": "wiki", "action": "created", - "key": "knowledge/global/revenue-gross-to-net.md" + "key": "wiki/global/revenue-gross-to-net.md" }, { "type": "candidate_action", "unitKey": "revenue-and-contracts", "target": "wiki", "action": "created", - "key": "knowledge/global/discount-expiration.md" + "key": "wiki/global/discount-expiration.md" }, { "type": "candidate_action", @@ -127,7 +127,7 @@ "type": "work_unit_started", "unitKey": "retention-and-segments", "skills": [ - "knowledge_capture", + "wiki_capture", "sl_capture" ], "stepBudget": 40 @@ -137,14 +137,14 @@ "unitKey": "retention-and-segments", "target": "wiki", "action": "created", - "key": "knowledge/global/nrr-retention.md" + "key": "wiki/global/nrr-retention.md" }, { "type": "candidate_action", "unitKey": "retention-and-segments", "target": "wiki", "action": "created", - "key": "knowledge/global/segment-classification.md" + "key": "wiki/global/segment-classification.md" }, { "type": "candidate_action", @@ -162,7 +162,7 @@ "type": "work_unit_started", "unitKey": "procurement-and-activation", "skills": [ - "knowledge_capture", + "wiki_capture", "sl_capture" ], "stepBudget": 40 @@ -172,14 +172,14 @@ "unitKey": "procurement-and-activation", "target": "wiki", "action": "created", - "key": "knowledge/global/activation-policy.md" + "key": "wiki/global/activation-policy.md" }, { "type": "candidate_action", "unitKey": "procurement-and-activation", "target": "wiki", "action": "created", - "key": "knowledge/global/procurement-workflows.md" + "key": "wiki/global/procurement-workflows.md" }, { "type": "candidate_action", @@ -197,7 +197,7 @@ "type": "work_unit_started", "unitKey": "support-and-health", "skills": [ - "knowledge_capture", + "wiki_capture", "sl_capture" ], "stepBudget": 40 @@ -207,14 +207,14 @@ "unitKey": "support-and-health", "target": "wiki", "action": "created", - "key": "knowledge/global/customer-health-scoring.md" + "key": "wiki/global/customer-health-scoring.md" }, { "type": "candidate_action", "unitKey": "support-and-health", "target": "wiki", "action": "created", - "key": "knowledge/global/support-escalation.md" + "key": "wiki/global/support-escalation.md" }, { "type": "candidate_action", @@ -232,7 +232,7 @@ "type": "work_unit_started", "unitKey": "governance-and-exclusions", "skills": [ - "knowledge_capture" + "wiki_capture" ], "stepBudget": 40 }, @@ -241,7 +241,7 @@ "unitKey": "governance-and-exclusions", "target": "wiki", "action": "created", - "key": "knowledge/global/internal-test-exclusion.md" + "key": "wiki/global/internal-test-exclusion.md" }, { "type": "work_unit_finished", @@ -321,7 +321,7 @@ "unitKey": "revenue-and-contracts", "target": "wiki", "action": "created", - "key": "knowledge/global/arr-contract-first.md", + "key": "wiki/global/arr-contract-first.md", "summary": "ARR follows contract precedence with cancellation and discount caveats.", "rawFiles": [ "contracts", @@ -334,7 +334,7 @@ "unitKey": "revenue-and-contracts", "target": "wiki", "action": "created", - "key": "knowledge/global/revenue-gross-to-net.md", + "key": "wiki/global/revenue-gross-to-net.md", "summary": "Invoice, refund, and revenue dashboard evidence reconcile gross to net revenue.", "rawFiles": [ "invoices", @@ -346,7 +346,7 @@ "unitKey": "revenue-and-contracts", "target": "wiki", "action": "created", - "key": "knowledge/global/discount-expiration.md", + "key": "wiki/global/discount-expiration.md", "summary": "Discount expiration is separated from organic contraction for retention reporting.", "rawFiles": [ "contracts", @@ -394,7 +394,7 @@ "unitKey": "retention-and-segments", "target": "wiki", "action": "created", - "key": "knowledge/global/nrr-retention.md", + "key": "wiki/global/nrr-retention.md", "summary": "NRR uses parent-account rollups and quarterly ARR movement windows.", "rawFiles": [ "accounts", @@ -407,7 +407,7 @@ "unitKey": "retention-and-segments", "target": "wiki", "action": "created", - "key": "knowledge/global/segment-classification.md", + "key": "wiki/global/segment-classification.md", "summary": "Segment labels come from plan mapping and sales-ops policy notes.", "rawFiles": [ "accounts", @@ -432,7 +432,7 @@ "unitKey": "procurement-and-activation", "target": "wiki", "action": "created", - "key": "knowledge/global/activation-policy.md", + "key": "wiki/global/activation-policy.md", "summary": "Activation policy changed on January 15, 2026 and is encoded for agents.", "rawFiles": [ "purchase_requests", @@ -445,7 +445,7 @@ "unitKey": "procurement-and-activation", "target": "wiki", "action": "created", - "key": "knowledge/global/procurement-workflows.md", + "key": "wiki/global/procurement-workflows.md", "summary": "Procurement requester activity and approval events explain product usage.", "rawFiles": [ "purchase_requests", @@ -468,7 +468,7 @@ "unitKey": "support-and-health", "target": "wiki", "action": "created", - "key": "knowledge/global/customer-health-scoring.md", + "key": "wiki/global/customer-health-scoring.md", "summary": "Customer health combines support severity, ARR exposure, and product usage.", "rawFiles": [ "support_tickets", @@ -480,7 +480,7 @@ "unitKey": "support-and-health", "target": "wiki", "action": "created", - "key": "knowledge/global/support-escalation.md", + "key": "wiki/global/support-escalation.md", "summary": "Escalation tiers map ticket severity to SLA expectations.", "rawFiles": [ "support_tickets", @@ -503,7 +503,7 @@ "unitKey": "governance-and-exclusions", "target": "wiki", "action": "created", - "key": "knowledge/global/internal-test-exclusion.md", + "key": "wiki/global/internal-test-exclusion.md", "summary": "Canonical metrics exclude internal and test accounts across source families.", "rawFiles": [ "raw-sources/notion/analyst-onboarding.md" @@ -515,97 +515,97 @@ { "rawPath": "contracts", "artifactKind": "wiki", - "artifactKey": "knowledge/global/arr-contract-first.md", + "artifactKey": "wiki/global/arr-contract-first.md", "actionType": "wiki_written" }, { "rawPath": "raw-sources/notion/arr-and-contract-reporting-notes.md", "artifactKind": "wiki", - "artifactKey": "knowledge/global/arr-contract-first.md", + "artifactKey": "wiki/global/arr-contract-first.md", "actionType": "wiki_written" }, { "rawPath": "invoices", "artifactKind": "wiki", - "artifactKey": "knowledge/global/revenue-gross-to-net.md", + "artifactKey": "wiki/global/revenue-gross-to-net.md", "actionType": "wiki_written" }, { "rawPath": "raw-sources/notion/revenue-reporting-policy.md", "artifactKind": "wiki", - "artifactKey": "knowledge/global/revenue-gross-to-net.md", + "artifactKey": "wiki/global/revenue-gross-to-net.md", "actionType": "wiki_written" }, { "rawPath": "arr_movements", "artifactKind": "wiki", - "artifactKey": "knowledge/global/discount-expiration.md", + "artifactKey": "wiki/global/discount-expiration.md", "actionType": "wiki_written" }, { "rawPath": "arr_movements", "artifactKind": "wiki", - "artifactKey": "knowledge/global/nrr-retention.md", + "artifactKey": "wiki/global/nrr-retention.md", "actionType": "wiki_written" }, { "rawPath": "raw-sources/notion/retention-and-nrr-definition-notes.md", "artifactKind": "wiki", - "artifactKey": "knowledge/global/nrr-retention.md", + "artifactKey": "wiki/global/nrr-retention.md", "actionType": "wiki_written" }, { "rawPath": "raw-sources/bi/account_retention.view.lkml", "artifactKind": "wiki", - "artifactKey": "knowledge/global/nrr-retention.md", + "artifactKey": "wiki/global/nrr-retention.md", "actionType": "wiki_written" }, { "rawPath": "plans", "artifactKind": "wiki", - "artifactKey": "knowledge/global/segment-classification.md", + "artifactKey": "wiki/global/segment-classification.md", "actionType": "wiki_written" }, { "rawPath": "raw-sources/notion/sales-ops-segmentation-guide.md", "artifactKind": "wiki", - "artifactKey": "knowledge/global/segment-classification.md", + "artifactKey": "wiki/global/segment-classification.md", "actionType": "wiki_written" }, { "rawPath": "raw-sources/notion/activation-policy-decision-record.md", "artifactKind": "wiki", - "artifactKey": "knowledge/global/activation-policy.md", + "artifactKey": "wiki/global/activation-policy.md", "actionType": "wiki_written" }, { "rawPath": "purchase_requests", "artifactKind": "wiki", - "artifactKey": "knowledge/global/procurement-workflows.md", + "artifactKey": "wiki/global/procurement-workflows.md", "actionType": "wiki_written" }, { "rawPath": "raw-sources/notion/customer-health-playbook.md", "artifactKind": "wiki", - "artifactKey": "knowledge/global/customer-health-scoring.md", + "artifactKey": "wiki/global/customer-health-scoring.md", "actionType": "wiki_written" }, { "rawPath": "support_tickets", "artifactKind": "wiki", - "artifactKey": "knowledge/global/customer-health-scoring.md", + "artifactKey": "wiki/global/customer-health-scoring.md", "actionType": "wiki_written" }, { "rawPath": "raw-sources/notion/support-escalation-runbook.md", "artifactKind": "wiki", - "artifactKey": "knowledge/global/support-escalation.md", + "artifactKey": "wiki/global/support-escalation.md", "actionType": "wiki_written" }, { "rawPath": "raw-sources/notion/analyst-onboarding.md", "artifactKind": "wiki", - "artifactKey": "knowledge/global/internal-test-exclusion.md", + "artifactKey": "wiki/global/internal-test-exclusion.md", "actionType": "wiki_written" }, { diff --git a/packages/cli/assets/demo/orbit/knowledge/global/.gitkeep b/packages/cli/assets/demo/orbit/wiki/global/.gitkeep similarity index 100% rename from packages/cli/assets/demo/orbit/knowledge/global/.gitkeep rename to packages/cli/assets/demo/orbit/wiki/global/.gitkeep diff --git a/packages/cli/assets/demo/orbit/knowledge/global/customer-communication-policy.md b/packages/cli/assets/demo/orbit/wiki/global/customer-communication-policy.md similarity index 100% rename from packages/cli/assets/demo/orbit/knowledge/global/customer-communication-policy.md rename to packages/cli/assets/demo/orbit/wiki/global/customer-communication-policy.md diff --git a/packages/cli/assets/demo/orbit/knowledge/global/new-hire-onboarding-policy.md b/packages/cli/assets/demo/orbit/wiki/global/new-hire-onboarding-policy.md similarity index 100% rename from packages/cli/assets/demo/orbit/knowledge/global/new-hire-onboarding-policy.md rename to packages/cli/assets/demo/orbit/wiki/global/new-hire-onboarding-policy.md diff --git a/packages/cli/assets/demo/orbit/knowledge/global/orbit-activation-kpi-glossary.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-activation-kpi-glossary.md similarity index 100% rename from packages/cli/assets/demo/orbit/knowledge/global/orbit-activation-kpi-glossary.md rename to packages/cli/assets/demo/orbit/wiki/global/orbit-activation-kpi-glossary.md diff --git a/packages/cli/assets/demo/orbit/knowledge/global/orbit-activation-policy-change-jan-2026.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-activation-policy-change-jan-2026.md similarity index 100% rename from packages/cli/assets/demo/orbit/knowledge/global/orbit-activation-policy-change-jan-2026.md rename to packages/cli/assets/demo/orbit/wiki/global/orbit-activation-policy-change-jan-2026.md diff --git a/packages/cli/assets/demo/orbit/knowledge/global/orbit-arr-contract-first-definition.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-arr-contract-first-definition.md similarity index 100% rename from packages/cli/assets/demo/orbit/knowledge/global/orbit-arr-contract-first-definition.md rename to packages/cli/assets/demo/orbit/wiki/global/orbit-arr-contract-first-definition.md diff --git a/packages/cli/assets/demo/orbit/knowledge/global/orbit-company-overview.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-company-overview.md similarity index 100% rename from packages/cli/assets/demo/orbit/knowledge/global/orbit-company-overview.md rename to packages/cli/assets/demo/orbit/wiki/global/orbit-company-overview.md diff --git a/packages/cli/assets/demo/orbit/knowledge/global/orbit-customer-health-risk-definition.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-customer-health-risk-definition.md similarity index 100% rename from packages/cli/assets/demo/orbit/knowledge/global/orbit-customer-health-risk-definition.md rename to packages/cli/assets/demo/orbit/wiki/global/orbit-customer-health-risk-definition.md diff --git a/packages/cli/assets/demo/orbit/knowledge/global/orbit-customer-stakeholder-needs.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-customer-stakeholder-needs.md similarity index 100% rename from packages/cli/assets/demo/orbit/knowledge/global/orbit-customer-stakeholder-needs.md rename to packages/cli/assets/demo/orbit/wiki/global/orbit-customer-stakeholder-needs.md diff --git a/packages/cli/assets/demo/orbit/knowledge/global/orbit-customers-source.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-customers-source.md similarity index 100% rename from packages/cli/assets/demo/orbit/knowledge/global/orbit-customers-source.md rename to packages/cli/assets/demo/orbit/wiki/global/orbit-customers-source.md diff --git a/packages/cli/assets/demo/orbit/knowledge/global/orbit-dbt-exposures.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-dbt-exposures.md similarity index 100% rename from packages/cli/assets/demo/orbit/knowledge/global/orbit-dbt-exposures.md rename to packages/cli/assets/demo/orbit/wiki/global/orbit-dbt-exposures.md diff --git a/packages/cli/assets/demo/orbit/knowledge/global/orbit-dbt-project-overview.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-dbt-project-overview.md similarity index 100% rename from packages/cli/assets/demo/orbit/knowledge/global/orbit-dbt-project-overview.md rename to packages/cli/assets/demo/orbit/wiki/global/orbit-dbt-project-overview.md diff --git a/packages/cli/assets/demo/orbit/knowledge/global/orbit-how-we-work.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-how-we-work.md similarity index 100% rename from packages/cli/assets/demo/orbit/knowledge/global/orbit-how-we-work.md rename to packages/cli/assets/demo/orbit/wiki/global/orbit-how-we-work.md diff --git a/packages/cli/assets/demo/orbit/knowledge/global/orbit-known-product-gaps.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-known-product-gaps.md similarity index 100% rename from packages/cli/assets/demo/orbit/knowledge/global/orbit-known-product-gaps.md rename to packages/cli/assets/demo/orbit/wiki/global/orbit-known-product-gaps.md diff --git a/packages/cli/assets/demo/orbit/knowledge/global/orbit-mart-account-activity.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-mart-account-activity.md similarity index 100% rename from packages/cli/assets/demo/orbit/knowledge/global/orbit-mart-account-activity.md rename to packages/cli/assets/demo/orbit/wiki/global/orbit-mart-account-activity.md diff --git a/packages/cli/assets/demo/orbit/knowledge/global/orbit-mart-account-segments.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-mart-account-segments.md similarity index 100% rename from packages/cli/assets/demo/orbit/knowledge/global/orbit-mart-account-segments.md rename to packages/cli/assets/demo/orbit/wiki/global/orbit-mart-account-segments.md diff --git a/packages/cli/assets/demo/orbit/knowledge/global/orbit-mart-arr-daily.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-mart-arr-daily.md similarity index 100% rename from packages/cli/assets/demo/orbit/knowledge/global/orbit-mart-arr-daily.md rename to packages/cli/assets/demo/orbit/wiki/global/orbit-mart-arr-daily.md diff --git a/packages/cli/assets/demo/orbit/knowledge/global/orbit-mart-nrr-quarterly.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-mart-nrr-quarterly.md similarity index 100% rename from packages/cli/assets/demo/orbit/knowledge/global/orbit-mart-nrr-quarterly.md rename to packages/cli/assets/demo/orbit/wiki/global/orbit-mart-nrr-quarterly.md diff --git a/packages/cli/assets/demo/orbit/knowledge/global/orbit-mart-procurement-activity.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-mart-procurement-activity.md similarity index 100% rename from packages/cli/assets/demo/orbit/knowledge/global/orbit-mart-procurement-activity.md rename to packages/cli/assets/demo/orbit/wiki/global/orbit-mart-procurement-activity.md diff --git a/packages/cli/assets/demo/orbit/knowledge/global/orbit-mart-retention-movement-breakout.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-mart-retention-movement-breakout.md similarity index 100% rename from packages/cli/assets/demo/orbit/knowledge/global/orbit-mart-retention-movement-breakout.md rename to packages/cli/assets/demo/orbit/wiki/global/orbit-mart-retention-movement-breakout.md diff --git a/packages/cli/assets/demo/orbit/knowledge/global/orbit-mart-revenue-daily.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-mart-revenue-daily.md similarity index 100% rename from packages/cli/assets/demo/orbit/knowledge/global/orbit-mart-revenue-daily.md rename to packages/cli/assets/demo/orbit/wiki/global/orbit-mart-revenue-daily.md diff --git a/packages/cli/assets/demo/orbit/knowledge/global/orbit-metabase-sql-library-patterns.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-metabase-sql-library-patterns.md similarity index 100% rename from packages/cli/assets/demo/orbit/knowledge/global/orbit-metabase-sql-library-patterns.md rename to packages/cli/assets/demo/orbit/wiki/global/orbit-metabase-sql-library-patterns.md diff --git a/packages/cli/assets/demo/orbit/knowledge/global/orbit-nrr-discount-expiration-treatment.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-nrr-discount-expiration-treatment.md similarity index 100% rename from packages/cli/assets/demo/orbit/knowledge/global/orbit-nrr-discount-expiration-treatment.md rename to packages/cli/assets/demo/orbit/wiki/global/orbit-nrr-discount-expiration-treatment.md diff --git a/packages/cli/assets/demo/orbit/knowledge/global/orbit-plan-segment-normalization.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-plan-segment-normalization.md similarity index 100% rename from packages/cli/assets/demo/orbit/knowledge/global/orbit-plan-segment-normalization.md rename to packages/cli/assets/demo/orbit/wiki/global/orbit-plan-segment-normalization.md diff --git a/packages/cli/assets/demo/orbit/knowledge/global/orbit-procurement-qualifying-actions.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-procurement-qualifying-actions.md similarity index 100% rename from packages/cli/assets/demo/orbit/knowledge/global/orbit-procurement-qualifying-actions.md rename to packages/cli/assets/demo/orbit/wiki/global/orbit-procurement-qualifying-actions.md diff --git a/packages/cli/assets/demo/orbit/knowledge/global/orbit-product-design-principles.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-product-design-principles.md similarity index 100% rename from packages/cli/assets/demo/orbit/knowledge/global/orbit-product-design-principles.md rename to packages/cli/assets/demo/orbit/wiki/global/orbit-product-design-principles.md diff --git a/packages/cli/assets/demo/orbit/knowledge/global/orbit-product-review-checklist.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-product-review-checklist.md similarity index 100% rename from packages/cli/assets/demo/orbit/knowledge/global/orbit-product-review-checklist.md rename to packages/cli/assets/demo/orbit/wiki/global/orbit-product-review-checklist.md diff --git a/packages/cli/assets/demo/orbit/knowledge/global/orbit-revenue-gross-to-net-reconciliation.md b/packages/cli/assets/demo/orbit/wiki/global/orbit-revenue-gross-to-net-reconciliation.md similarity index 100% rename from packages/cli/assets/demo/orbit/knowledge/global/orbit-revenue-gross-to-net-reconciliation.md rename to packages/cli/assets/demo/orbit/wiki/global/orbit-revenue-gross-to-net-reconciliation.md diff --git a/packages/cli/assets/demo/orbit/knowledge/global/sales-ops-cs-handoff-process.md b/packages/cli/assets/demo/orbit/wiki/global/sales-ops-cs-handoff-process.md similarity index 100% rename from packages/cli/assets/demo/orbit/knowledge/global/sales-ops-cs-handoff-process.md rename to packages/cli/assets/demo/orbit/wiki/global/sales-ops-cs-handoff-process.md diff --git a/packages/cli/scripts/build-demo-assets.mjs b/packages/cli/scripts/build-demo-assets.mjs index 2e135b1c..82f611a2 100644 --- a/packages/cli/scripts/build-demo-assets.mjs +++ b/packages/cli/scripts/build-demo-assets.mjs @@ -229,39 +229,39 @@ const knowledgePages = [ ]; const provenanceLinks = [ - ['wiki', 'knowledge/global/arr-contract-first.md', 'warehouse', 'contracts', 'describes', 1], + ['wiki', 'wiki/global/arr-contract-first.md', 'warehouse', 'contracts', 'describes', 1], [ 'wiki', - 'knowledge/global/arr-contract-first.md', + 'wiki/global/arr-contract-first.md', 'notion', 'raw-sources/notion/arr-and-contract-reporting-notes.md', 'derived_from', 0.95, ], - ['wiki', 'knowledge/global/revenue-gross-to-net.md', 'warehouse', 'invoices', 'describes', 1], + ['wiki', 'wiki/global/revenue-gross-to-net.md', 'warehouse', 'invoices', 'describes', 1], [ 'wiki', - 'knowledge/global/revenue-gross-to-net.md', + 'wiki/global/revenue-gross-to-net.md', 'notion', 'raw-sources/notion/revenue-reporting-policy.md', 'derived_from', 0.95, ], - ['wiki', 'knowledge/global/discount-expiration.md', 'warehouse', 'arr_movements', 'describes', 1], - ['wiki', 'knowledge/global/nrr-retention.md', 'warehouse', 'arr_movements', 'describes', 1], + ['wiki', 'wiki/global/discount-expiration.md', 'warehouse', 'arr_movements', 'describes', 1], + ['wiki', 'wiki/global/nrr-retention.md', 'warehouse', 'arr_movements', 'describes', 1], [ 'wiki', - 'knowledge/global/nrr-retention.md', + 'wiki/global/nrr-retention.md', 'notion', 'raw-sources/notion/retention-and-nrr-definition-notes.md', 'derived_from', 0.95, ], - ['wiki', 'knowledge/global/nrr-retention.md', 'bi', 'raw-sources/bi/account_retention.view.lkml', 'derived_from', 0.85], - ['wiki', 'knowledge/global/segment-classification.md', 'warehouse', 'plans', 'describes', 1], + ['wiki', 'wiki/global/nrr-retention.md', 'bi', 'raw-sources/bi/account_retention.view.lkml', 'derived_from', 0.85], + ['wiki', 'wiki/global/segment-classification.md', 'warehouse', 'plans', 'describes', 1], [ 'wiki', - 'knowledge/global/segment-classification.md', + 'wiki/global/segment-classification.md', 'notion', 'raw-sources/notion/sales-ops-segmentation-guide.md', 'derived_from', @@ -269,25 +269,25 @@ const provenanceLinks = [ ], [ 'wiki', - 'knowledge/global/activation-policy.md', + 'wiki/global/activation-policy.md', 'notion', 'raw-sources/notion/activation-policy-decision-record.md', 'derived_from', 0.95, ], - ['wiki', 'knowledge/global/procurement-workflows.md', 'warehouse', 'purchase_requests', 'describes', 1], + ['wiki', 'wiki/global/procurement-workflows.md', 'warehouse', 'purchase_requests', 'describes', 1], [ 'wiki', - 'knowledge/global/customer-health-scoring.md', + 'wiki/global/customer-health-scoring.md', 'notion', 'raw-sources/notion/customer-health-playbook.md', 'derived_from', 0.9, ], - ['wiki', 'knowledge/global/customer-health-scoring.md', 'warehouse', 'support_tickets', 'describes', 1], + ['wiki', 'wiki/global/customer-health-scoring.md', 'warehouse', 'support_tickets', 'describes', 1], [ 'wiki', - 'knowledge/global/support-escalation.md', + 'wiki/global/support-escalation.md', 'notion', 'raw-sources/notion/support-escalation-runbook.md', 'derived_from', @@ -295,7 +295,7 @@ const provenanceLinks = [ ], [ 'wiki', - 'knowledge/global/internal-test-exclusion.md', + 'wiki/global/internal-test-exclusion.md', 'notion', 'raw-sources/notion/analyst-onboarding.md', 'derived_from', @@ -490,7 +490,7 @@ function buildActions() { unitKey: 'revenue-and-contracts', target: 'wiki', action: 'created', - key: 'knowledge/global/arr-contract-first.md', + key: 'wiki/global/arr-contract-first.md', summary: 'ARR follows contract precedence with cancellation and discount caveats.', rawFiles: ['contracts', 'arr_movements', 'raw-sources/notion/arr-and-contract-reporting-notes.md'], status: 'success', @@ -499,7 +499,7 @@ function buildActions() { unitKey: 'revenue-and-contracts', target: 'wiki', action: 'created', - key: 'knowledge/global/revenue-gross-to-net.md', + key: 'wiki/global/revenue-gross-to-net.md', summary: 'Invoice, refund, and revenue dashboard evidence reconcile gross to net revenue.', rawFiles: ['invoices', 'raw-sources/bi/revenue_exec.dashboard.lookml'], status: 'success', @@ -508,7 +508,7 @@ function buildActions() { unitKey: 'revenue-and-contracts', target: 'wiki', action: 'created', - key: 'knowledge/global/discount-expiration.md', + key: 'wiki/global/discount-expiration.md', summary: 'Discount expiration is separated from organic contraction for retention reporting.', rawFiles: ['contracts', 'arr_movements'], status: 'success', @@ -544,7 +544,7 @@ function buildActions() { unitKey: 'retention-and-segments', target: 'wiki', action: 'created', - key: 'knowledge/global/nrr-retention.md', + key: 'wiki/global/nrr-retention.md', summary: 'NRR uses parent-account rollups and quarterly ARR movement windows.', rawFiles: ['accounts', 'arr_movements', 'raw-sources/notion/retention-and-nrr-definition-notes.md'], status: 'success', @@ -553,7 +553,7 @@ function buildActions() { unitKey: 'retention-and-segments', target: 'wiki', action: 'created', - key: 'knowledge/global/segment-classification.md', + key: 'wiki/global/segment-classification.md', summary: 'Segment labels come from plan mapping and sales-ops policy notes.', rawFiles: ['accounts', 'plans', 'raw-sources/notion/sales-ops-segmentation-guide.md'], status: 'success', @@ -571,7 +571,7 @@ function buildActions() { unitKey: 'procurement-and-activation', target: 'wiki', action: 'created', - key: 'knowledge/global/activation-policy.md', + key: 'wiki/global/activation-policy.md', summary: 'Activation policy changed on January 15, 2026 and is encoded for agents.', rawFiles: ['purchase_requests', 'users', 'raw-sources/notion/activation-policy-decision-record.md'], status: 'success', @@ -580,7 +580,7 @@ function buildActions() { unitKey: 'procurement-and-activation', target: 'wiki', action: 'created', - key: 'knowledge/global/procurement-workflows.md', + key: 'wiki/global/procurement-workflows.md', summary: 'Procurement requester activity and approval events explain product usage.', rawFiles: ['purchase_requests', 'raw-sources/bi/procurement_activity.view.lkml'], status: 'success', @@ -598,7 +598,7 @@ function buildActions() { unitKey: 'support-and-health', target: 'wiki', action: 'created', - key: 'knowledge/global/customer-health-scoring.md', + key: 'wiki/global/customer-health-scoring.md', summary: 'Customer health combines support severity, ARR exposure, and product usage.', rawFiles: ['support_tickets', 'raw-sources/notion/customer-health-playbook.md'], status: 'success', @@ -607,7 +607,7 @@ function buildActions() { unitKey: 'support-and-health', target: 'wiki', action: 'created', - key: 'knowledge/global/support-escalation.md', + key: 'wiki/global/support-escalation.md', summary: 'Escalation tiers map ticket severity to SLA expectations.', rawFiles: ['support_tickets', 'raw-sources/notion/support-escalation-runbook.md'], status: 'success', @@ -625,7 +625,7 @@ function buildActions() { unitKey: 'governance-and-exclusions', target: 'wiki', action: 'created', - key: 'knowledge/global/internal-test-exclusion.md', + key: 'wiki/global/internal-test-exclusion.md', summary: 'Canonical metrics exclude internal and test accounts across source families.', rawFiles: ['raw-sources/notion/analyst-onboarding.md'], status: 'success', @@ -665,27 +665,27 @@ function buildReplay(provenance, transcripts) { { type: 'raw_snapshot_written', syncId: 'demo-seeded-sync', rawFileCount: 29 }, { type: 'diff_computed', added: 29, modified: 0, deleted: 0, unchanged: 0 }, { type: 'chunks_planned', chunkCount: 5, workUnitCount: 5, evictionCount: 0 }, - { type: 'work_unit_started', unitKey: 'revenue-and-contracts', skills: ['knowledge_capture', 'sl_capture'], stepBudget: 40 }, + { type: 'work_unit_started', unitKey: 'revenue-and-contracts', skills: ['wiki_capture', 'sl_capture'], stepBudget: 40 }, { type: 'candidate_action', unitKey: 'revenue-and-contracts', target: 'wiki', action: 'created', - key: 'knowledge/global/arr-contract-first.md', + key: 'wiki/global/arr-contract-first.md', }, { type: 'candidate_action', unitKey: 'revenue-and-contracts', target: 'wiki', action: 'created', - key: 'knowledge/global/revenue-gross-to-net.md', + key: 'wiki/global/revenue-gross-to-net.md', }, { type: 'candidate_action', unitKey: 'revenue-and-contracts', target: 'wiki', action: 'created', - key: 'knowledge/global/discount-expiration.md', + key: 'wiki/global/discount-expiration.md', }, { type: 'candidate_action', @@ -709,20 +709,20 @@ function buildReplay(provenance, transcripts) { key: 'orbit_demo.arr_movements', }, { type: 'work_unit_finished', unitKey: 'revenue-and-contracts', status: 'success' }, - { type: 'work_unit_started', unitKey: 'retention-and-segments', skills: ['knowledge_capture', 'sl_capture'], stepBudget: 40 }, + { type: 'work_unit_started', unitKey: 'retention-and-segments', skills: ['wiki_capture', 'sl_capture'], stepBudget: 40 }, { type: 'candidate_action', unitKey: 'retention-and-segments', target: 'wiki', action: 'created', - key: 'knowledge/global/nrr-retention.md', + key: 'wiki/global/nrr-retention.md', }, { type: 'candidate_action', unitKey: 'retention-and-segments', target: 'wiki', action: 'created', - key: 'knowledge/global/segment-classification.md', + key: 'wiki/global/segment-classification.md', }, { type: 'candidate_action', @@ -735,7 +735,7 @@ function buildReplay(provenance, transcripts) { { type: 'work_unit_started', unitKey: 'procurement-and-activation', - skills: ['knowledge_capture', 'sl_capture'], + skills: ['wiki_capture', 'sl_capture'], stepBudget: 40, }, { @@ -743,14 +743,14 @@ function buildReplay(provenance, transcripts) { unitKey: 'procurement-and-activation', target: 'wiki', action: 'created', - key: 'knowledge/global/activation-policy.md', + key: 'wiki/global/activation-policy.md', }, { type: 'candidate_action', unitKey: 'procurement-and-activation', target: 'wiki', action: 'created', - key: 'knowledge/global/procurement-workflows.md', + key: 'wiki/global/procurement-workflows.md', }, { type: 'candidate_action', @@ -760,20 +760,20 @@ function buildReplay(provenance, transcripts) { key: 'orbit_demo.purchase_requests', }, { type: 'work_unit_finished', unitKey: 'procurement-and-activation', status: 'success' }, - { type: 'work_unit_started', unitKey: 'support-and-health', skills: ['knowledge_capture', 'sl_capture'], stepBudget: 40 }, + { type: 'work_unit_started', unitKey: 'support-and-health', skills: ['wiki_capture', 'sl_capture'], stepBudget: 40 }, { type: 'candidate_action', unitKey: 'support-and-health', target: 'wiki', action: 'created', - key: 'knowledge/global/customer-health-scoring.md', + key: 'wiki/global/customer-health-scoring.md', }, { type: 'candidate_action', unitKey: 'support-and-health', target: 'wiki', action: 'created', - key: 'knowledge/global/support-escalation.md', + key: 'wiki/global/support-escalation.md', }, { type: 'candidate_action', @@ -783,13 +783,13 @@ function buildReplay(provenance, transcripts) { key: 'orbit_demo.support_tickets', }, { type: 'work_unit_finished', unitKey: 'support-and-health', status: 'success' }, - { type: 'work_unit_started', unitKey: 'governance-and-exclusions', skills: ['knowledge_capture'], stepBudget: 40 }, + { type: 'work_unit_started', unitKey: 'governance-and-exclusions', skills: ['wiki_capture'], stepBudget: 40 }, { type: 'candidate_action', unitKey: 'governance-and-exclusions', target: 'wiki', action: 'created', - key: 'knowledge/global/internal-test-exclusion.md', + key: 'wiki/global/internal-test-exclusion.md', }, { type: 'work_unit_finished', unitKey: 'governance-and-exclusions', status: 'success' }, { type: 'reconciliation_finished', conflictCount: 0, fallbackCount: 0 }, @@ -835,7 +835,7 @@ function buildReplay(provenance, transcripts) { async function writeGeneratedContext(rowCounts) { for (const page of knowledgePages) { - await writeText(join('knowledge/global', page.file), renderKnowledgePage(page)); + await writeText(join('wiki/global', page.file), renderKnowledgePage(page)); } for (const table of semanticLayerTables) { @@ -908,7 +908,7 @@ async function writeGeneratedContext(rowCounts) { }, generated: { semanticLayer: { path: 'semantic-layer/orbit_demo', sourceCount: 6 }, - knowledge: { path: 'knowledge/global', pageCount: 10 }, + knowledge: { path: 'wiki/global', pageCount: 10 }, links: { path: 'links', linkCount: provenanceLinks.length }, }, }); @@ -930,7 +930,7 @@ for (const relativeDir of [ 'raw-sources/bi', 'raw-sources/notion', 'semantic-layer/orbit_demo', - 'knowledge/global', + 'wiki/global', 'links', 'reports', ]) { diff --git a/packages/cli/src/command-schemas.ts b/packages/cli/src/command-schemas.ts index e1365d86..5caece1f 100644 --- a/packages/cli/src/command-schemas.ts +++ b/packages/cli/src/command-schemas.ts @@ -3,6 +3,19 @@ import { z } from 'zod'; const projectDirSchema = z.string().min(1); const stringArraySchema = z.array(z.string()); +export const wikiWriteCommandSchema = z.object({ + command: z.literal('write'), + projectDir: projectDirSchema, + key: z.string().min(1), + scope: z.enum(['GLOBAL', 'USER']), + userId: z.string().min(1), + summary: z.string().min(1), + content: z.string().min(1), + tags: stringArraySchema, + refs: stringArraySchema, + slRefs: stringArraySchema, +}); + const orderBySchema = z.union([ z.string().min(1), z.object({ diff --git a/packages/cli/src/commands/knowledge-commands.ts b/packages/cli/src/commands/knowledge-commands.ts index 382ebf0a..f8d716f7 100644 --- a/packages/cli/src/commands/knowledge-commands.ts +++ b/packages/cli/src/commands/knowledge-commands.ts @@ -1,9 +1,11 @@ -import { type Command } from '@commander-js/extra-typings'; +import { type Command, Option } from '@commander-js/extra-typings'; import { + collectOption, type KtxCliCommandContext, parsePositiveIntegerOption, resolveCommandProjectDir, } from '../cli-program.js'; +import { wikiWriteCommandSchema } from '../command-schemas.js'; import type { KtxKnowledgeArgs } from '../knowledge.js'; import { profileMark } from '../startup-profile.js'; @@ -17,7 +19,7 @@ async function runKnowledgeArgs(context: KtxCliCommandContext, args: KtxKnowledg export function registerWikiCommands(program: Command, context: KtxCliCommandContext): void { const wiki = program .command('wiki') - .description('List or search local wiki pages') + .description('List, read, search, or write local wiki pages') .showHelpAfterError() .addHelpText( 'after', @@ -38,6 +40,22 @@ export function registerWikiCommands(program: Command, context: KtxCliCommandCon }); }); + wiki + .command('read') + .description('Read one local wiki page') + .argument('', 'Wiki page key') + .option('--json', 'Print JSON output', false) + .option('--user-id ', 'Local user id', 'local') + .action(async (key: string, options: { userId: string; json?: boolean }, command) => { + await runKnowledgeArgs(context, { + command: 'read', + projectDir: resolveCommandProjectDir(command), + key, + userId: options.userId, + json: options.json, + }); + }); + wiki .command('search') .description('Search local wiki pages') @@ -55,4 +73,31 @@ export function registerWikiCommands(program: Command, context: KtxCliCommandCon ...(options.limit !== undefined ? { limit: options.limit } : {}), }); }); + + wiki + .command('write') + .description('Write one local wiki page') + .argument('', 'Wiki page key') + .option('--user-id ', 'Local user id', 'local') + .addOption(new Option('--scope ', 'global or user').choices(['global', 'user']).default('global')) + .requiredOption('--summary ', 'Wiki summary') + .requiredOption('--content ', 'Wiki content') + .option('--tag ', 'Wiki tag; repeatable', collectOption, []) + .option('--ref ', 'Wiki ref; repeatable', collectOption, []) + .option('--sl-ref ', 'Semantic-layer ref; repeatable', collectOption, []) + .action(async (key: string, options, command) => { + const args = wikiWriteCommandSchema.parse({ + command: 'write', + projectDir: resolveCommandProjectDir(command), + key, + scope: options.scope === 'user' ? 'USER' : 'GLOBAL', + userId: options.userId, + summary: options.summary, + content: options.content, + tags: options.tag, + refs: options.ref, + slRefs: options.slRef, + }); + await runKnowledgeArgs(context, args); + }); } diff --git a/packages/cli/src/demo-assets.test.ts b/packages/cli/src/demo-assets.test.ts index 575e9bb7..92aad645 100644 --- a/packages/cli/src/demo-assets.test.ts +++ b/packages/cli/src/demo-assets.test.ts @@ -95,7 +95,7 @@ describe('demo assets', () => { await expect(access(packagedDemoAssetPath('semantic-layer/dbt-main/mart_arr_daily.yaml'))).resolves.toBeUndefined(); await expect(access(packagedDemoAssetPath('semantic-layer/postgres-warehouse/mart_account_activity.yaml'))).resolves.toBeUndefined(); - await expect(access(packagedDemoAssetPath('knowledge/global/orbit-company-overview.md'))).resolves.toBeUndefined(); + await expect(access(packagedDemoAssetPath('wiki/global/orbit-company-overview.md'))).resolves.toBeUndefined(); await expect(access(packagedDemoAssetPath('links/provenance.json'))).resolves.toBeUndefined(); await expect(access(packagedDemoAssetPath('reports/seeded-demo-report.json'))).resolves.toBeUndefined(); }); @@ -108,7 +108,7 @@ describe('demo assets', () => { await expect(access(join(projectDir, 'state.sqlite'))).resolves.toBeUndefined(); await expect(access(join(projectDir, 'reports'))).resolves.toBeUndefined(); await expect(access(join(projectDir, 'semantic-layer'))).resolves.toBeUndefined(); - await expect(access(join(projectDir, 'knowledge'))).resolves.toBeUndefined(); + await expect(access(join(projectDir, 'wiki'))).resolves.toBeUndefined(); await expect(access(join(projectDir, 'replays', 'replay.memory-flow.v1.json'))).resolves.toBeUndefined(); await expect(access(join(projectDir, 'raw-sources'))).resolves.toBeUndefined(); await expect(access(join(projectDir, '_schema'))).rejects.toMatchObject({ code: 'ENOENT' }); @@ -129,7 +129,7 @@ describe('demo assets', () => { await ensureSeededDemoProject({ projectDir, force: false }); await expect(access(join(projectDir, 'semantic-layer', 'dbt-main', 'mart_arr_daily.yaml'))).resolves.toBeUndefined(); - await expect(access(join(projectDir, 'knowledge', 'global', 'orbit-company-overview.md'))).resolves.toBeUndefined(); + await expect(access(join(projectDir, 'wiki', 'global', 'orbit-company-overview.md'))).resolves.toBeUndefined(); await expect(access(join(projectDir, 'links', 'provenance.json'))).resolves.toBeUndefined(); await expect(access(join(projectDir, 'reports', 'seeded-demo-report.json'))).resolves.toBeUndefined(); }); diff --git a/packages/cli/src/demo-assets.ts b/packages/cli/src/demo-assets.ts index 4bab5ead..1e972ef7 100644 --- a/packages/cli/src/demo-assets.ts +++ b/packages/cli/src/demo-assets.ts @@ -29,7 +29,7 @@ const REQUIRED_SEEDED_ASSET_PATHS = [ DEMO_REPLAY_FILE, join('semantic-layer', 'dbt-main', 'mart_arr_daily.yaml'), join('semantic-layer', 'postgres-warehouse', 'mart_account_activity.yaml'), - join('knowledge', 'global', 'orbit-company-overview.md'), + join('wiki', 'global', 'orbit-company-overview.md'), ] as const; function assetDir(): string { @@ -131,7 +131,7 @@ export async function ensureDemoProject(options: EnsureDemoProjectOptions): Prom } await mkdir(projectDir, { recursive: true }); - for (const relativeDir of ['reports', 'semantic-layer', 'knowledge', 'replays', 'raw-sources', 'links']) { + for (const relativeDir of ['reports', 'semantic-layer', 'wiki', 'replays', 'raw-sources', 'links']) { await mkdir(join(projectDir, relativeDir), { recursive: true }); } @@ -157,7 +157,7 @@ async function copySeededAssetDirectories(projectDir: string): Promise { await Promise.all([ copyDirIfExists(join(src, 'semantic-layer'), join(dest, 'semantic-layer')), - copyDirIfExists(join(src, 'knowledge'), join(dest, 'knowledge')), + copyDirIfExists(join(src, 'wiki'), join(dest, 'wiki')), copyDirIfExists(join(src, 'raw-sources'), join(dest, 'raw-sources')), copyDirIfExists(join(src, 'links'), join(dest, 'links')), copyDirIfExists(join(src, 'reports'), join(dest, 'reports')), diff --git a/packages/cli/src/index.test.ts b/packages/cli/src/index.test.ts index f41f4b6a..f914a875 100644 --- a/packages/cli/src/index.test.ts +++ b/packages/cli/src/index.test.ts @@ -139,22 +139,78 @@ describe('runKtxCli', () => { expect(testIo.stderr()).toBe(''); }); - it('rejects removed public wiki and sl read/write commands', async () => { - const sl = vi.fn(async () => 0); + it('routes public wiki read and write commands', async () => { const knowledge = vi.fn(async () => 0); + const readIo = makeIo(); + await expect(runKtxCli(['--project-dir', tempDir, 'wiki', 'read', 'revenue', '--json'], readIo.io, { knowledge })) + .resolves.toBe(0); + expect(knowledge).toHaveBeenCalledWith( + { + command: 'read', + projectDir: tempDir, + key: 'revenue', + userId: 'local', + json: true, + }, + readIo.io, + ); + + const writeIo = makeIo(); + await expect( + runKtxCli( + [ + '--project-dir', + tempDir, + 'wiki', + 'write', + 'revenue', + '--scope', + 'user', + '--summary', + 'Revenue', + '--content', + 'Revenue.', + '--tag', + 'finance', + '--ref', + 'https://example.com/revenue', + '--sl-ref', + 'orders', + ], + writeIo.io, + { knowledge }, + ), + ).resolves.toBe(0); + expect(knowledge).toHaveBeenLastCalledWith( + { + command: 'write', + projectDir: tempDir, + key: 'revenue', + scope: 'USER', + userId: 'local', + summary: 'Revenue', + content: 'Revenue.', + tags: ['finance'], + refs: ['https://example.com/revenue'], + slRefs: ['orders'], + }, + writeIo.io, + ); + }); + + it('rejects removed public sl read/write commands', async () => { + const sl = vi.fn(async () => 0); + for (const argv of [ - ['--project-dir', tempDir, 'wiki', 'read', 'revenue'], - ['--project-dir', tempDir, 'wiki', 'write', 'revenue', '--summary', 'Revenue', '--content', 'Revenue.'], ['--project-dir', tempDir, 'sl', 'read', 'orders', '--connection-id', 'warehouse'], ['--project-dir', tempDir, 'sl', 'write', 'orders', '--connection-id', 'warehouse', '--yaml', 'name: orders'], ]) { const io = makeIo(); - await expect(runKtxCli(argv, io.io, { knowledge, sl })).resolves.toBe(1); + await expect(runKtxCli(argv, io.io, { sl })).resolves.toBe(1); expect(io.stderr()).toMatch(/unknown command|error:/); } - expect(knowledge).not.toHaveBeenCalled(); expect(sl).not.toHaveBeenCalled(); }); diff --git a/packages/cli/src/ingest.test-utils.ts b/packages/cli/src/ingest.test-utils.ts index 73190b0d..7b65e33a 100644 --- a/packages/cli/src/ingest.test-utils.ts +++ b/packages/cli/src/ingest.test-utils.ts @@ -159,7 +159,7 @@ export function bundleReportSnapshot(): IngestReportSnapshot { rawFiles: ['cards/1.json', 'cards/2.json'], status: 'success', actions: [ - { target: 'wiki', type: 'created', key: 'knowledge/global/revenue.md', detail: 'Revenue overview' }, + { target: 'wiki', type: 'created', key: 'wiki/global/revenue.md', detail: 'Revenue overview' }, { target: 'sl', type: 'updated', key: 'warehouse.orders', detail: 'Added order amount measure' }, ], touchedSlSources: [{ connectionId: 'warehouse', sourceName: 'warehouse.orders' }], @@ -178,7 +178,7 @@ export function bundleReportSnapshot(): IngestReportSnapshot { { rawPath: 'cards/1.json', artifactKind: 'wiki', - artifactKey: 'knowledge/global/revenue.md', + artifactKey: 'wiki/global/revenue.md', actionType: 'wiki_written', }, { @@ -194,7 +194,7 @@ export function bundleReportSnapshot(): IngestReportSnapshot { path: 'tool-transcripts/cards.jsonl', toolCallCount: 4, errorCount: 0, - toolNames: ['ingest_triage', 'knowledge_capture', 'sl_capture'], + toolNames: ['ingest_triage', 'wiki_capture', 'sl_capture'], }, ], }, diff --git a/packages/cli/src/knowledge.test.ts b/packages/cli/src/knowledge.test.ts index 1982fe1c..c4b3fdd9 100644 --- a/packages/cli/src/knowledge.test.ts +++ b/packages/cli/src/knowledge.test.ts @@ -3,7 +3,6 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { initKtxProject } from '@ktx/context/project'; import type { KtxEmbeddingPort } from '@ktx/context'; -import { type LocalKnowledgeScope, writeLocalKnowledgePage } from '@ktx/context/wiki'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { runKtxKnowledge } from './knowledge.js'; @@ -41,29 +40,6 @@ class FakeEmbeddingPort implements KtxEmbeddingPort { } } -async function seedKnowledgePage(input: { - projectDir: string; - key: string; - summary: string; - content: string; - scope?: LocalKnowledgeScope; - tags?: string[]; - refs?: string[]; - slRefs?: string[]; -}): Promise { - const project = await initKtxProject({ projectDir: input.projectDir, projectName: 'warehouse' }); - await writeLocalKnowledgePage(project, { - key: input.key, - scope: input.scope ?? 'GLOBAL', - userId: 'local', - summary: input.summary, - content: input.content, - tags: input.tags ?? [], - refs: input.refs ?? [], - slRefs: input.slRefs ?? [], - }); -} - describe('runKtxKnowledge', () => { let tempDir: string; @@ -75,16 +51,36 @@ describe('runKtxKnowledge', () => { await rm(tempDir, { recursive: true, force: true }); }); - it('lists and searches knowledge pages', async () => { + it('writes, reads, lists, and searches wiki pages', async () => { const projectDir = join(tempDir, 'project'); - await seedKnowledgePage({ - projectDir, - key: 'metrics-revenue', - summary: 'Revenue', - content: 'Revenue is paid order value.', - tags: ['finance'], - slRefs: ['orders'], - }); + await initKtxProject({ projectDir, projectName: 'warehouse' }); + + const writeIo = makeIo(); + await expect( + runKtxKnowledge( + { + command: 'write', + projectDir, + key: 'metrics-revenue', + scope: 'GLOBAL', + userId: 'local', + summary: 'Revenue', + content: 'Revenue is paid order value.', + tags: ['finance'], + refs: [], + slRefs: ['orders'], + }, + writeIo.io, + ), + ).resolves.toBe(0); + expect(writeIo.stdout()).toContain('Wrote wiki/global/metrics-revenue.md'); + + const readIo = makeIo(); + await expect( + runKtxKnowledge({ command: 'read', projectDir, key: 'metrics-revenue', userId: 'local' }, readIo.io), + ).resolves.toBe(0); + expect(readIo.stdout()).toContain('# metrics-revenue'); + expect(readIo.stdout()).toContain('Revenue is paid order value.'); const listIo = makeIo(); await expect(runKtxKnowledge({ command: 'list', projectDir, userId: 'local' }, listIo.io)).resolves.toBe(0); @@ -97,16 +93,27 @@ describe('runKtxKnowledge', () => { expect(searchIo.stdout()).toContain('metrics-revenue'); }); - it('prints wiki list and search as public JSON envelopes', async () => { + it('prints wiki list, search, and read as public JSON envelopes', async () => { const projectDir = join(tempDir, 'project'); - await seedKnowledgePage({ - projectDir, - key: 'metrics-revenue', - summary: 'Revenue', - content: 'Revenue is paid order value.', - tags: ['finance'], - slRefs: ['orders'], - }); + await initKtxProject({ projectDir, projectName: 'warehouse' }); + + await expect( + runKtxKnowledge( + { + command: 'write', + projectDir, + key: 'metrics-revenue', + scope: 'GLOBAL', + userId: 'local', + summary: 'Revenue', + content: 'Revenue is paid order value.', + tags: ['finance'], + refs: [], + slRefs: ['orders'], + }, + makeIo().io, + ), + ).resolves.toBe(0); const listIo = makeIo(); await expect(runKtxKnowledge({ command: 'list', projectDir, userId: 'local', json: true }, listIo.io)).resolves.toBe( @@ -130,6 +137,48 @@ describe('runKtxKnowledge', () => { data: { items: [expect.objectContaining({ key: 'metrics-revenue', summary: 'Revenue' })] }, meta: { command: 'wiki search' }, }); + + const readIo = makeIo(); + await expect( + runKtxKnowledge({ command: 'read', projectDir, key: 'metrics-revenue', userId: 'local', json: true }, readIo.io), + ).resolves.toBe(0); + expect(JSON.parse(readIo.stdout())).toMatchObject({ + kind: 'wiki.page', + data: { + key: 'metrics-revenue', + summary: 'Revenue', + content: 'Revenue is paid order value.', + }, + }); + }); + + it('rejects slash-delimited write keys with a flat-key suggestion', async () => { + const projectDir = join(tempDir, 'project'); + await initKtxProject({ projectDir, projectName: 'warehouse' }); + + const writeIo = makeIo(); + await expect( + runKtxKnowledge( + { + command: 'write', + projectDir, + key: 'orbit/company-overview', + scope: 'GLOBAL', + userId: 'local', + summary: 'Orbit', + content: 'Orbit overview.', + tags: [], + refs: [], + slRefs: [], + }, + writeIo.io, + ), + ).resolves.toBe(1); + + expect(writeIo.stderr()).toContain( + 'Invalid wiki key "orbit/company-overview". Wiki keys must be flat; use "orbit-company-overview".', + ); + expect(writeIo.stdout()).toBe(''); }); it('explains empty search results for a project without wiki pages', async () => { @@ -143,19 +192,30 @@ describe('runKtxKnowledge', () => { expect(searchIo.stdout()).toBe(''); expect(searchIo.stderr()).toContain('No local wiki pages found'); - expect(searchIo.stderr()).toContain('Run ingest'); - expect(searchIo.stderr()).not.toContain('ktx wiki write'); + expect(searchIo.stderr()).toContain('ktx wiki write'); }); it('uses configured embeddings for semantic wiki search', async () => { const projectDir = join(tempDir, 'semantic-project'); - await seedKnowledgePage({ - projectDir, - key: 'active-contract-arr-open-tickets', - summary: 'Active Contract ARR Ranked by Open Support Ticket Count', - content: 'Accounts ranked by annual recurring contract value and support ticket load.', - tags: ['historic-sql'], - }); + await initKtxProject({ projectDir, projectName: 'warehouse' }); + + await expect( + runKtxKnowledge( + { + command: 'write', + projectDir, + key: 'active-contract-arr-open-tickets', + scope: 'GLOBAL', + userId: 'local', + summary: 'Active Contract ARR Ranked by Open Support Ticket Count', + content: 'Accounts ranked by annual recurring contract value and support ticket load.', + tags: ['historic-sql'], + refs: [], + slRefs: [], + }, + makeIo().io, + ), + ).resolves.toBe(0); const searchIo = makeIo(); await expect( diff --git a/packages/cli/src/knowledge.ts b/packages/cli/src/knowledge.ts index 0d1e194b..2e039dea 100644 --- a/packages/cli/src/knowledge.ts +++ b/packages/cli/src/knowledge.ts @@ -4,12 +4,31 @@ import { type KtxEmbeddingPort, } from '@ktx/context'; import { loadKtxProject } from '@ktx/context/project'; -import { listLocalKnowledgePages, searchLocalKnowledgePages } from '@ktx/context/wiki'; +import { + type LocalKnowledgeScope, + listLocalKnowledgePages, + readLocalKnowledgePage, + searchLocalKnowledgePages, + writeLocalKnowledgePage, +} from '@ktx/context/wiki'; import { writeJsonResult } from './io/print-list.js'; export type KtxKnowledgeArgs = | { command: 'list'; projectDir: string; userId: string; json?: boolean } - | { command: 'search'; projectDir: string; query: string; userId: string; json?: boolean; limit?: number }; + | { command: 'read'; projectDir: string; key: string; userId: string; json?: boolean } + | { command: 'search'; projectDir: string; query: string; userId: string; json?: boolean; limit?: number } + | { + command: 'write'; + projectDir: string; + key: string; + scope: LocalKnowledgeScope; + userId: string; + summary: string; + content: string; + tags: string[]; + refs: string[]; + slRefs: string[]; + }; interface KtxKnowledgeIo { stdout: { write(chunk: string): void }; @@ -56,6 +75,25 @@ export async function runKtxKnowledge( } return 0; } + if (args.command === 'read') { + const page = await readLocalKnowledgePage(project, { key: args.key, userId: args.userId }); + if (!page) { + throw new Error(`Wiki page "${args.key}" was not found`); + } + if (args.json) { + writeJsonResult(io, { + kind: 'wiki.page', + data: page, + meta: { command: 'wiki read' }, + }); + return 0; + } + io.stdout.write(`# ${page.key}\n\n`); + io.stdout.write(`Scope: ${page.scope}\n`); + io.stdout.write(`Summary: ${page.summary}\n\n`); + io.stdout.write(`${page.content}\n`); + return 0; + } if (args.command === 'search') { const results = await searchLocalKnowledgePages(project, { query: args.query, @@ -75,7 +113,7 @@ export async function runKtxKnowledge( const pages = await listLocalKnowledgePages(project, { userId: args.userId }); if (pages.length === 0) { io.stderr.write( - `No local wiki pages found in ${project.projectDir}. Run ingest to capture wiki context, then retry the search.\n`, + `No local wiki pages found in ${project.projectDir}. Create one with \`ktx wiki write --summary --content \` or run ingest.\n`, ); } else { io.stderr.write( @@ -89,8 +127,19 @@ export async function runKtxKnowledge( } return 0; } - const _exhaustive: never = args; - throw new Error(`Unsupported wiki command: ${JSON.stringify(_exhaustive)}`); + + const write = await writeLocalKnowledgePage(project, { + key: args.key, + scope: args.scope, + userId: args.userId, + summary: args.summary, + content: args.content, + tags: args.tags, + refs: args.refs, + slRefs: args.slRefs, + }); + io.stdout.write(`Wrote ${write.path}\n`); + return 0; } catch (error) { io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); return 1; diff --git a/packages/cli/src/memory-flow-hud.tsx b/packages/cli/src/memory-flow-hud.tsx index 5d2be9eb..9a9b3d96 100644 --- a/packages/cli/src/memory-flow-hud.tsx +++ b/packages/cli/src/memory-flow-hud.tsx @@ -76,7 +76,7 @@ function tableName(key: string): string { function humanizeInsight(key: string, target: 'sl' | 'wiki', summary: string | undefined): string { if (summary) return summary; const name = target === 'sl' ? tableName(key) : topicName(key); - return target === 'sl' ? `Query definition: ${name}` : `Knowledge page: ${name}`; + return target === 'sl' ? `Query definition: ${name}` : `Wiki page: ${name}`; } const INTERNAL_DEMO_CONNECTION_ID = 'orbit_demo'; @@ -453,7 +453,7 @@ function CompletionSummary(props: { )} {wiki > 0 && ( - {' '}📝 {wiki} knowledge page{wiki === 1 ? '' : 's'} — so agents understand your business context + {' '}📝 {wiki} wiki page{wiki === 1 ? '' : 's'} — so agents understand your business context )} diff --git a/packages/cli/src/memory-flow-interactive.test.ts b/packages/cli/src/memory-flow-interactive.test.ts index 456a3110..d7fe8bd8 100644 --- a/packages/cli/src/memory-flow-interactive.test.ts +++ b/packages/cli/src/memory-flow-interactive.test.ts @@ -46,9 +46,9 @@ function replay(): MemoryFlowReplayInput { { type: 'raw_snapshot_written', syncId: 'sync-1', rawFileCount: 2 }, { type: 'diff_computed', added: 1, modified: 1, deleted: 0, unchanged: 0 }, { type: 'chunks_planned', chunkCount: 2, workUnitCount: 2, evictionCount: 0 }, - { type: 'work_unit_started', unitKey: 'orders', skills: ['knowledge_capture'], stepBudget: 4 }, + { type: 'work_unit_started', unitKey: 'orders', skills: ['wiki_capture'], stepBudget: 4 }, { type: 'work_unit_finished', unitKey: 'orders', status: 'success' }, - { type: 'work_unit_started', unitKey: 'customers', skills: ['knowledge_capture'], stepBudget: 4 }, + { type: 'work_unit_started', unitKey: 'customers', skills: ['wiki_capture'], stepBudget: 4 }, { type: 'work_unit_finished', unitKey: 'customers', status: 'failed', reason: 'validation reset' }, { type: 'reconciliation_finished', conflictCount: 0, fallbackCount: 1 }, { type: 'saved', commitSha: 'abc12345', wikiCount: 1, slCount: 1 }, diff --git a/packages/cli/src/memory-flow-tui.test.tsx b/packages/cli/src/memory-flow-tui.test.tsx index b555c6c1..e1df900a 100644 --- a/packages/cli/src/memory-flow-tui.test.tsx +++ b/packages/cli/src/memory-flow-tui.test.tsx @@ -23,10 +23,10 @@ function replayInput(): MemoryFlowReplayInput { ], details: { actions: [ - { unitKey: 'orders', target: 'wiki', action: 'created', key: 'knowledge/orders.md', summary: 'order lifecycle', rawFiles: ['orders'], status: 'success' }, + { unitKey: 'orders', target: 'wiki', action: 'created', key: 'wiki/orders.md', summary: 'order lifecycle', rawFiles: ['orders'], status: 'success' }, { unitKey: 'customers', target: 'sl', action: 'updated', key: 'orbit_demo.customers', summary: 'customer metrics', rawFiles: ['customers'], status: 'success' }, ], - provenance: [{ rawPath: 'orders', artifactKind: 'wiki', artifactKey: 'knowledge/orders.md', actionType: 'wiki_written' }], + provenance: [{ rawPath: 'orders', artifactKind: 'wiki', artifactKey: 'wiki/orders.md', actionType: 'wiki_written' }], transcripts: [{ unitKey: 'orders', path: '/tmp/t.jsonl', toolCallCount: 2, errorCount: 0, toolNames: ['read_raw_span', 'wiki_write'] }], }, events: [ @@ -35,8 +35,8 @@ function replayInput(): MemoryFlowReplayInput { { type: 'raw_snapshot_written', syncId: 'sync-1', rawFileCount: 2 }, { type: 'diff_computed', added: 1, modified: 1, deleted: 0, unchanged: 0 }, { type: 'chunks_planned', chunkCount: 2, workUnitCount: 2, evictionCount: 0 }, - { type: 'work_unit_started', unitKey: 'orders', skills: ['knowledge_capture'], stepBudget: 40 }, - { type: 'candidate_action', unitKey: 'orders', target: 'wiki', action: 'created', key: 'knowledge/orders.md' }, + { type: 'work_unit_started', unitKey: 'orders', skills: ['wiki_capture'], stepBudget: 40 }, + { type: 'candidate_action', unitKey: 'orders', target: 'wiki', action: 'created', key: 'wiki/orders.md' }, { type: 'work_unit_finished', unitKey: 'orders', status: 'success' }, { type: 'work_unit_started', unitKey: 'customers', skills: ['sl_capture'], stepBudget: 40 }, { type: 'candidate_action', unitKey: 'customers', target: 'sl', action: 'updated', key: 'orbit_demo.customers' }, @@ -220,7 +220,7 @@ describe('MemoryFlowTuiApp', () => { { type: 'source_acquired', adapter: 'live-database', trigger: 'manual_resync', fileCount: 1 }, { type: 'diff_computed', added: 1, modified: 0, deleted: 0, unchanged: 0 }, { type: 'chunks_planned', chunkCount: 1, workUnitCount: 1, evictionCount: 0 }, - { type: 'work_unit_started', unitKey: 'orders', skills: ['knowledge_capture'], stepBudget: 40 }, + { type: 'work_unit_started', unitKey: 'orders', skills: ['wiki_capture'], stepBudget: 40 }, ], plannedWorkUnits: [{ unitKey: 'orders', rawFiles: ['orders'], peerFileCount: 0, dependencyCount: 1 }], }; @@ -240,7 +240,7 @@ describe('MemoryFlowTuiApp', () => { { type: 'source_acquired', adapter: 'dbt-descriptions', trigger: 'manual_resync', fileCount: 3 }, { type: 'diff_computed', added: 11, modified: 0, deleted: 0, unchanged: 0 }, { type: 'chunks_planned', chunkCount: 1, workUnitCount: 1, evictionCount: 0 }, - { type: 'work_unit_started', unitKey: 'orders', skills: ['knowledge_capture'], stepBudget: 40 }, + { type: 'work_unit_started', unitKey: 'orders', skills: ['wiki_capture'], stepBudget: 40 }, ], plannedWorkUnits: [{ unitKey: 'orders', rawFiles: ['orders'], peerFileCount: 0, dependencyCount: 1 }], }; diff --git a/packages/cli/src/setup-context.test.ts b/packages/cli/src/setup-context.test.ts index 9115d7a5..1c2ab320 100644 --- a/packages/cli/src/setup-context.test.ts +++ b/packages/cli/src/setup-context.test.ts @@ -257,9 +257,9 @@ describe('setup context build state', () => { it('marks context complete without prompting when initial source ingest already made agent context', async () => { await writeReadyProject(tempDir); await mkdir(join(tempDir, 'semantic-layer', 'dbt-main'), { recursive: true }); - await mkdir(join(tempDir, 'knowledge', 'global'), { recursive: true }); + await mkdir(join(tempDir, 'wiki', 'global'), { recursive: true }); await writeFile(join(tempDir, 'semantic-layer', 'dbt-main', 'mart_revenue_daily.yaml'), 'name: mart_revenue_daily\n'); - await writeFile(join(tempDir, 'knowledge', 'global', 'metrics.md'), '# Metrics\n'); + await writeFile(join(tempDir, 'wiki', 'global', 'metrics.md'), '# Metrics\n'); await writeReadyEnrichedScanReport(tempDir); const io = makeIo(); const runContextBuildMock = vi.fn(async () => ({ exitCode: 0, detached: false })); @@ -332,8 +332,8 @@ describe('setup context build state', () => { await writeFile(join(tempDir, 'semantic-layer', 'warehouse', '_schema', 'public.yaml'), 'tables: {}\n'); const io = makeIo(); const runContextBuildMock = vi.fn(async () => { - await mkdir(join(tempDir, 'knowledge', 'global'), { recursive: true }); - await writeFile(join(tempDir, 'knowledge', 'global', 'metrics.md'), '# Metrics\n'); + await mkdir(join(tempDir, 'wiki', 'global'), { recursive: true }); + await writeFile(join(tempDir, 'wiki', 'global', 'metrics.md'), '# Metrics\n'); await writeReadyEnrichedScanReport(tempDir); return { exitCode: 0, detached: false }; }); diff --git a/packages/cli/src/setup-context.ts b/packages/cli/src/setup-context.ts index 94589bdc..ea23a9dc 100644 --- a/packages/cli/src/setup-context.ts +++ b/packages/cli/src/setup-context.ts @@ -441,7 +441,7 @@ async function defaultVerifyContextReady(projectDir: string): Promise { expect(options).toContainEqual({ value: 'notion', label: 'Notion' }); }); + it('shows already configured context sources in the interactive checklist', async () => { + await addPrimarySource(); + await addConnection('notion-main', { + driver: 'notion', + auth_token_ref: 'env:NOTION_TOKEN', + crawl_mode: 'all_accessible', + }); + const io = makeIo(); + const testPrompts = prompts({ multiselect: [['back']] }); + + await expect( + runKtxSetupSourcesStep( + { projectDir, inputMode: 'auto', runInitialSourceIngest: false, skipSources: false }, + io.io, + { prompts: testPrompts }, + ), + ).resolves.toEqual({ status: 'back', projectDir }); + + expect(testPrompts.multiselect).toHaveBeenCalledWith( + expect.objectContaining({ + initialValues: ['notion'], + options: expect.arrayContaining([{ value: 'notion', label: 'Notion', hint: 'configured: notion-main' }]), + }), + ); + }); + it('uses a source-specific editable connection name for new interactive connections', async () => { await addPrimarySource(); const validateDbt = vi.fn(async () => ({ ok: true as const, detail: 'project=analytics schemas=2' })); diff --git a/packages/cli/src/setup-sources.ts b/packages/cli/src/setup-sources.ts index 6ab71106..0561b0e2 100644 --- a/packages/cli/src/setup-sources.ts +++ b/packages/cli/src/setup-sources.ts @@ -73,7 +73,8 @@ export type KtxSetupSourcesResult = export interface KtxSetupSourcesPromptAdapter { multiselect(options: { message: string; - options: Array<{ value: string; label: string }>; + options: Array<{ value: string; label: string; hint?: string }>; + initialValues?: string[]; required?: boolean; }): Promise; select(options: { message: string; options: Array<{ value: string; label: string }> }): Promise; @@ -1325,6 +1326,22 @@ function existingConnectionIdsBySource( .sort((left, right) => left.localeCompare(right)); } +function sourceChecklistForConnections(connections: Record): { + options: Array<{ value: KtxSetupSourceType; label: string; hint?: string }>; + initialValues: KtxSetupSourceType[]; +} { + const initialValues: KtxSetupSourceType[] = []; + const options = SOURCE_OPTIONS.map((option) => { + const existingIds = existingConnectionIdsBySource(connections, option.value); + if (existingIds.length === 0) { + return option; + } + initialValues.push(option.value); + return { ...option, hint: `configured: ${existingIds.join(', ')}` }; + }); + return { options, initialValues }; +} + function defaultConnectionIdForSource( connections: Record, source: KtxSetupSourceType, @@ -1483,13 +1500,19 @@ export async function runKtxSetupSourcesStep( } while (true) { + const contextSourceChecklist = sourceChecklistForConnections( + (await loadKtxProject({ projectDir: args.projectDir })).config.connections, + ); const selected = args.source ? [args.source] : args.inputMode === 'disabled' ? [] : await prompts.multiselect({ message: withMultiselectNavigation('Which context sources should KTX ingest?'), - options: [...SOURCE_OPTIONS], + options: contextSourceChecklist.options, + ...(contextSourceChecklist.initialValues.length > 0 + ? { initialValues: contextSourceChecklist.initialValues } + : {}), required: false, }); if (selected.includes('back')) { diff --git a/packages/context/prompts/memory_agent_backfill.md b/packages/context/prompts/memory_agent_backfill.md index ee0f7ed4..fdf7211d 100644 --- a/packages/context/prompts/memory_agent_backfill.md +++ b/packages/context/prompts/memory_agent_backfill.md @@ -10,7 +10,7 @@ Capture only when the signal is unambiguous: a metric definition stated plainly, 1. Read the wiki and SL indexes to avoid creating duplicates. -2. If the content has wiki-style signal, load the `knowledge_capture` skill and follow its workflow. +2. If the content has wiki-style signal, load the `wiki_capture` skill and follow its workflow. 3. If the content has SL-style signal, load the `sl` skill and follow its Part 3 workflow. 4. Prefer updating existing entries over creating new ones — backfills often duplicate existing knowledge. 5. When done, exit the loop. diff --git a/packages/context/prompts/memory_agent_bundle_ingest_reconcile.md b/packages/context/prompts/memory_agent_bundle_ingest_reconcile.md index 30b52537..515fecd3 100644 --- a/packages/context/prompts/memory_agent_bundle_ingest_reconcile.md +++ b/packages/context/prompts/memory_agent_bundle_ingest_reconcile.md @@ -7,7 +7,7 @@ Parsimonious. Stage 3 WUs already loaded `ingest_triage` and handled conflicts t -1. Load `ingest_triage`, then `sl_capture` + `knowledge_capture`. +1. Load `ingest_triage`, then `sl_capture` + `wiki_capture`. 2. Call `stage_list()` for the full index of this job's writes. If it is empty AND you have no evictions, exit — the runner short-circuits this case but the skill still teaches you to bail fast. 3. If the system prompt includes ``, apply those pins before flagging a same-name or near-duplicate conflict. A pinned `canonicalArtifactKey` keeps the contested name when it is present in the Stage Index; competing variants keep or receive disambiguated names. 4. Sweep both exact-key conflicts and near-duplicate writes. Compare WUs that wrote overlapping SL source names, overlapping wiki keys, the same `tables:` or `sl_refs:` action details, or obviously equivalent topic titles under different wiki keys. Call `stage_diff` to see the actual difference, and use `wiki_read`/`sl_read_source` when two different keys appear to describe the same table, metric, or source-of-truth mapping. If they're the same content, leave one canonical artifact and record the duplicate as subsumed. If they differ per `ingest_triage` rules, apply the correct resolution (rename + capture; election of canonical; silent replace for expression-only re-ingest change; or pinned canonical), then call `emit_conflict_resolution` with the artifact key and decision. diff --git a/packages/context/prompts/memory_agent_bundle_ingest_work_unit.md b/packages/context/prompts/memory_agent_bundle_ingest_work_unit.md index c7c9eb6d..3821537d 100644 --- a/packages/context/prompts/memory_agent_bundle_ingest_work_unit.md +++ b/packages/context/prompts/memory_agent_bundle_ingest_work_unit.md @@ -8,7 +8,7 @@ Assertive. The bundle was explicitly submitted for ingest. Default to capturing 1. Read this WorkUnit's section at the end of the user prompt. It lists your `rawFiles`, any unchanged `dependencyPaths` you may need to resolve references, the `peerFileIndex` (paths only; you CANNOT read them), the source's `skillNames`, and any `priorProvenance` rows telling you what earlier syncs produced from these files. -2. Load the per-source review skill first (e.g. `lookml_ingest`, `metricflow_ingest`, `dbt_ingest`), then `sl_capture` and `knowledge_capture`, and `ingest_triage` last. The triage skill tells you how to react when `discover_data` reveals that a prior WU already wrote something overlapping. +2. Load the per-source review skill first (e.g. `lookml_ingest`, `metricflow_ingest`, `dbt_ingest`), then `sl_capture` and `wiki_capture`, and `ingest_triage` last. The triage skill tells you how to react when `discover_data` reveals that a prior WU already wrote something overlapping. 3. If the system prompt includes ``, read those pins before choosing artifact keys. A pin's `canonicalArtifactKey` is the preferred artifact for its `contestedKey`: prefer editing the pinned canonical artifact when it already exists or when this raw file clearly updates it. Do not create a duplicate contested artifact when a pin says another artifact is canonical; use a specific disambiguated key only when the raw file describes a genuinely different domain. 4. For each raw file: call `read_raw_file` (or `read_raw_span` for slicing large files) to load content. Before writing a new SL source or wiki page, call `discover_data` for each candidate source, table, metric, or topic name to find prior-WU writes, existing wiki pages, SL sources, and raw warehouse matches; apply `ingest_triage` when you hit one, and apply any matching canonical pin before deciding whether to edit, rename, or skip. 5. For every `wiki_write`, `wiki_remove`, `sl_write_source`, or `sl_edit_source` call, include `rawPaths` with only the raw file paths that directly support that action. If one artifact synthesizes several files, list each contributing raw file. Do not include unrelated files from the same WorkUnit. diff --git a/packages/context/prompts/memory_agent_external_ingest.md b/packages/context/prompts/memory_agent_external_ingest.md index edee6f75..dd84651a 100644 --- a/packages/context/prompts/memory_agent_external_ingest.md +++ b/packages/context/prompts/memory_agent_external_ingest.md @@ -10,7 +10,7 @@ A single artifact typically produces multiple actions: one SL source per table/v 1. Review the wiki and SL indexes in the prompt. Prefer updating existing entries over creating duplicates. -2. Load the `sl` skill for SL-writes and `knowledge_capture` for wiki-writes. Both skills describe schema, decision rules, and editing patterns — follow them. +2. Load the `sl` skill for SL-writes and `wiki_capture` for wiki-writes. Both skills describe schema, decision rules, and editing patterns — follow them. 3. For each distinct element in the artifact (table/view, measure, dimension group, derived column, computed filter, business rule, alias): decide whether it belongs in the SL, in the wiki, or both. 4. Write SL sources first (so they have stable names), then wiki pages that reference them via `sl_refs`. 5. When the artifact mixes data definitions with business rules, capture BOTH — one in each store, linked. diff --git a/packages/context/prompts/memory_agent_research.md b/packages/context/prompts/memory_agent_research.md index f8a59a79..6090e5bb 100644 --- a/packages/context/prompts/memory_agent_research.md +++ b/packages/context/prompts/memory_agent_research.md @@ -19,7 +19,7 @@ Skip: 1. Read the wiki index and the SL sources index in the prompt below. 2. Identify durable knowledge OR reusable data patterns in the turn. -3. If the turn has wiki-style signal (preferences, definitions, conventions), load the `knowledge_capture` skill and follow its workflow. +3. If the turn has wiki-style signal (preferences, definitions, conventions), load the `wiki_capture` skill and follow its workflow. 4. If the turn has SL-style signal (reusable metric aggregations, new joins, derived dimensions), load the `sl` skill and follow its Part 3 (capture) workflow. 5. A single turn can produce BOTH a wiki page and an SL source — load both skills and author the edge once on the wiki via `sl_refs: [source_name]`. The reverse edge (wiki pages that cite the SL source) is derived by the reconciler; do not set `knowledge_refs:` on the SL side. 6. When you're done, exit the loop without calling any more tools. Do NOT emit a final text summary. diff --git a/packages/context/skills/metricflow_ingest/SKILL.md b/packages/context/skills/metricflow_ingest/SKILL.md index 6ed4b916..67743892 100644 --- a/packages/context/skills/metricflow_ingest/SKILL.md +++ b/packages/context/skills/metricflow_ingest/SKILL.md @@ -140,7 +140,7 @@ metrics: ``` Do NOT emit SL for this. Instead: -- Write a wiki page at `knowledge/global/-intent.md` quoting the full YAML body and a one-line explanation of the intended semantics (base event → conversion event within window). +- Write a wiki page at `wiki/global/-intent.md` quoting the full YAML body and a one-line explanation of the intended semantics (base event → conversion event within window). - Call `emit_unmapped_fallback` with `rawPath` set to the MetricFlow file path, `reason: "conversion_metric_unsupported"`, and `fallback: "flagged"`. When KTX SL gains conversion primitives, re-ingesting will find the prior wiki note (via `priorProvenance`) and replace it with an SL source. @@ -290,7 +290,7 @@ measures: - {name: margin, expr: "sum(revenue_cents) - sum(cost_cents)"} ``` -Also write a wiki page at `knowledge/global/margin-metric.md` explaining the cross-source origin. +Also write a wiki page at `wiki/global/margin-metric.md` explaining the cross-source origin. ## Example 4 — filtered metric creates a new measure diff --git a/packages/context/skills/knowledge_capture/SKILL.md b/packages/context/skills/wiki_capture/SKILL.md similarity index 97% rename from packages/context/skills/knowledge_capture/SKILL.md rename to packages/context/skills/wiki_capture/SKILL.md index 2a111d90..30188be6 100644 --- a/packages/context/skills/knowledge_capture/SKILL.md +++ b/packages/context/skills/wiki_capture/SKILL.md @@ -1,10 +1,10 @@ --- -name: knowledge_capture -description: KTX's knowledge base — wiki pages for durable, reusable business knowledge. Covers capture workflow for user preferences, metric definitions, organizational conventions, and cross-references between knowledge pages and semantic-layer sources. Loaded by the post-turn memory-agent only. The research agent reads wiki via `wiki_read`/`wiki_search` but does not write it. +name: wiki_capture +description: KTX's knowledge base — wiki pages for durable, reusable business knowledge. Covers capture workflow for user preferences, metric definitions, organizational conventions, and cross-references between wiki pages and semantic-layer sources. Loaded by the post-turn memory-agent only. The research agent reads wiki via `wiki_read`/`wiki_search` but does not write it. callers: [memory_agent] --- -# Knowledge Capture +# Wiki Capture ## Role diff --git a/packages/context/src/ingest/action-identity.test.ts b/packages/context/src/ingest/action-identity.test.ts index 0c855c41..725a1d99 100644 --- a/packages/context/src/ingest/action-identity.test.ts +++ b/packages/context/src/ingest/action-identity.test.ts @@ -19,13 +19,13 @@ describe('memory action target identity', () => { { target: 'wiki', type: 'created', - key: 'knowledge/global/orders.md', + key: 'wiki/global/orders.md', detail: '', targetConnectionId: 'ignored', }, 'looker-run', ), - ).toBe('wiki:looker-run:knowledge/global/orders.md'); + ).toBe('wiki:looker-run:wiki/global/orders.md'); }); it('resolves action target connection only for SL actions', () => { diff --git a/packages/context/src/ingest/adapters/dbt/dbt.adapter.ts b/packages/context/src/ingest/adapters/dbt/dbt.adapter.ts index ef1c798c..cdf1a434 100644 --- a/packages/context/src/ingest/adapters/dbt/dbt.adapter.ts +++ b/packages/context/src/ingest/adapters/dbt/dbt.adapter.ts @@ -16,7 +16,7 @@ interface DbtSourceAdapterOptions { export class DbtSourceAdapter implements SourceAdapter { readonly source = 'dbt' as const; - /** Runner merges: ingest_triage, sl_capture, knowledge_capture (see ingest-bundle.runner.ts) */ + /** Runner merges: ingest_triage, sl_capture, wiki_capture (see ingest-bundle.runner.ts) */ readonly skillNames: string[] = ['dbt_ingest']; constructor(private readonly options: DbtSourceAdapterOptions = {}) {} diff --git a/packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts b/packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts index c7a334bf..8f583d9c 100644 --- a/packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts +++ b/packages/context/src/ingest/adapters/historic-sql/local-ingest-acceptance.test.ts @@ -277,7 +277,7 @@ describe('historic-SQL local ingest retrieval acceptance', () => { await expect(readFile(join(project.projectDir, 'semantic-layer/warehouse/_schema/public.yaml'), 'utf-8')).resolves .toContain('Analysts repeatedly inspect paid order lifecycle by customer segment.'); - await expect(readFile(join(project.projectDir, 'knowledge/global/historic-sql-paid-order-lifecycle.md'), 'utf-8')) + await expect(readFile(join(project.projectDir, 'wiki/global/historic-sql-paid-order-lifecycle.md'), 'utf-8')) .resolves.toContain('Paid Order Lifecycle'); const reloaded = await loadKtxProject({ projectDir: project.projectDir }); diff --git a/packages/context/src/ingest/adapters/historic-sql/post-processor.ts b/packages/context/src/ingest/adapters/historic-sql/post-processor.ts index 8d89d397..f5e0aaec 100644 --- a/packages/context/src/ingest/adapters/historic-sql/post-processor.ts +++ b/packages/context/src/ingest/adapters/historic-sql/post-processor.ts @@ -10,7 +10,7 @@ async function commitProjectionChanges(workdir: string): Promise { const status = await git.status(); const paths = status.files .map((file) => file.path) - .filter((path) => path.startsWith('semantic-layer/') || path.startsWith('knowledge/global/historic-sql')); + .filter((path) => path.startsWith('semantic-layer/') || path.startsWith('wiki/global/historic-sql')); if (paths.length === 0) { return; } diff --git a/packages/context/src/ingest/adapters/historic-sql/projection.test.ts b/packages/context/src/ingest/adapters/historic-sql/projection.test.ts index 95adf13f..0b3c5604 100644 --- a/packages/context/src/ingest/adapters/historic-sql/projection.test.ts +++ b/packages/context/src/ingest/adapters/historic-sql/projection.test.ts @@ -106,7 +106,7 @@ describe('projectHistoricSqlEvidence', () => { await writeJson(workdir, 'raw-sources/warehouse/historic-sql/sync-1/tables/public.customers.json', { table: 'public.customers' }); await writeText( workdir, - 'knowledge/global/historic-sql-old-order-lifecycle.md', + 'wiki/global/historic-sql-old-order-lifecycle.md', [ '---', YAML.stringify({ @@ -127,7 +127,7 @@ describe('projectHistoricSqlEvidence', () => { ); await writeText( workdir, - 'knowledge/global/historic-sql-retired-pattern.md', + 'wiki/global/historic-sql-retired-pattern.md', [ '---', YAML.stringify({ @@ -164,10 +164,10 @@ describe('projectHistoricSqlEvidence', () => { const result = await projectHistoricSqlEvidence({ workdir, connectionId: 'warehouse', syncId: 'sync-1', runId: 'run-1' }); expect(result.patternPagesWritten).toBe(1); - await expect(readFile(join(workdir, 'knowledge/global/historic-sql-old-order-lifecycle.md'), 'utf-8')).resolves.toContain( + await expect(readFile(join(workdir, 'wiki/global/historic-sql-old-order-lifecycle.md'), 'utf-8')).resolves.toContain( 'Order Lifecycle Analysis', ); - await expect(readFile(join(workdir, 'knowledge/global/historic-sql-retired-pattern.md'), 'utf-8')).resolves.toContain( + await expect(readFile(join(workdir, 'wiki/global/historic-sql-retired-pattern.md'), 'utf-8')).resolves.toContain( 'stale_since: "2026-05-11T00:00:00.000Z"', ); }); @@ -192,7 +192,7 @@ describe('projectHistoricSqlEvidence', () => { await writeJson(workdir, 'raw-sources/warehouse/historic-sql/sync-1/tables/public.customers.json', { table: 'public.customers' }); await writeText( workdir, - 'knowledge/global/historic-sql-order-lifecycle-analysis.md', + 'wiki/global/historic-sql-order-lifecycle-analysis.md', [ '---', YAML.stringify({ @@ -230,7 +230,7 @@ describe('projectHistoricSqlEvidence', () => { const result = await projectHistoricSqlEvidence({ workdir, connectionId: 'warehouse', syncId: 'sync-1', runId: 'run-1' }); expect(result.patternPagesWritten).toBe(1); - const page = await readFile(join(workdir, 'knowledge/global/historic-sql-order-lifecycle-analysis.md'), 'utf-8'); + const page = await readFile(join(workdir, 'wiki/global/historic-sql-order-lifecycle-analysis.md'), 'utf-8'); expect(page).toContain('Analysts compare order status with customer segment again.'); expect(page).not.toContain('Archived body'); expect(page).not.toContain('archived'); @@ -254,7 +254,7 @@ describe('projectHistoricSqlEvidence', () => { }); await writeText( workdir, - 'knowledge/global/historic-sql-retired-pattern.md', + 'wiki/global/historic-sql-retired-pattern.md', [ '---', YAML.stringify({ @@ -279,7 +279,7 @@ describe('projectHistoricSqlEvidence', () => { expect(result.archivedPatternPages).toBe(0); expect(result.stalePatternPagesMarked).toBe(0); - await expect(readFile(join(workdir, 'knowledge/global/historic-sql-retired-pattern.md'), 'utf-8')).resolves.toContain( + await expect(readFile(join(workdir, 'wiki/global/historic-sql-retired-pattern.md'), 'utf-8')).resolves.toContain( 'Archived retired body', ); }); @@ -322,7 +322,7 @@ describe('projectHistoricSqlEvidence', () => { }); await writeText( workdir, - 'knowledge/global/historic-sql-old-template.md', + 'wiki/global/historic-sql-old-template.md', [ '---', YAML.stringify({ @@ -356,7 +356,7 @@ describe('projectHistoricSqlEvidence', () => { commonJoins: [], staleSince: '2026-05-11T00:00:00.000Z', }); - await expect(readFile(join(workdir, 'knowledge/global/historic-sql-old-template.md'), 'utf-8')).resolves.toContain( + await expect(readFile(join(workdir, 'wiki/global/historic-sql-old-template.md'), 'utf-8')).resolves.toContain( 'Old body', ); }); diff --git a/packages/context/src/ingest/adapters/historic-sql/projection.ts b/packages/context/src/ingest/adapters/historic-sql/projection.ts index 7d4da94f..36a7be19 100644 --- a/packages/context/src/ingest/adapters/historic-sql/projection.ts +++ b/packages/context/src/ingest/adapters/historic-sql/projection.ts @@ -276,7 +276,7 @@ export async function projectHistoricSqlEvidence(input: HistoricSqlProjectionInp } } - const wikiRoot = join(input.workdir, 'knowledge/global'); + const wikiRoot = join(input.workdir, 'wiki/global'); await mkdir(wikiRoot, { recursive: true }); const allPages = await loadPatternPages(wikiRoot); const activePages = allPages.filter((page) => !isArchivedPatternPage(page)); diff --git a/packages/context/src/ingest/ingest-bundle.runner.test.ts b/packages/context/src/ingest/ingest-bundle.runner.test.ts index b337a3f0..b9831c0f 100644 --- a/packages/context/src/ingest/ingest-bundle.runner.test.ts +++ b/packages/context/src/ingest/ingest-bundle.runner.test.ts @@ -599,7 +599,7 @@ describe('IngestBundleRunner — Stages 1 → 7', () => { currentToolSession.actions.push({ target: 'wiki', type: 'created', - key: 'knowledge/orders.md', + key: 'wiki/orders.md', detail: 'captured order context', }); } @@ -638,7 +638,7 @@ describe('IngestBundleRunner — Stages 1 → 7', () => { expect.objectContaining({ type: 'work_unit_started', unitKey: 'u1', - skills: ['ingest_triage', 'sl_capture', 'knowledge_capture'], + skills: ['ingest_triage', 'sl_capture', 'wiki_capture'], stepBudget: 40, }), expect.objectContaining({ type: 'work_unit_step', unitKey: 'u1', stepIndex: 1, stepBudget: 40 }), @@ -647,7 +647,7 @@ describe('IngestBundleRunner — Stages 1 → 7', () => { unitKey: 'u1', target: 'wiki', action: 'created', - key: 'knowledge/orders.md', + key: 'wiki/orders.md', }), expect.objectContaining({ type: 'work_unit_finished', unitKey: 'u1', status: 'success' }), ]), @@ -860,7 +860,7 @@ describe('IngestBundleRunner — Stages 1 → 7', () => { { toolCallId: 'ledger-1', messages: [] }, ); await params.toolSet.wiki_write.execute( - { key: 'knowledge/a.md', content: 'safe summary' }, + { key: 'wiki/a.md', content: 'safe summary' }, { toolCallId: 'wiki-1', messages: [] }, ); } @@ -1351,7 +1351,7 @@ describe('IngestBundleRunner — Stages 1 → 7', () => { { target: 'wiki', type: 'created', - key: 'knowledge/global/pipeline.md', + key: 'wiki/global/pipeline.md', detail: 'Pipeline article', }, { @@ -1391,7 +1391,7 @@ describe('IngestBundleRunner — Stages 1 → 7', () => { }); expect(deps.knowledgeSlRefs.syncFromWiki).toHaveBeenCalledWith({ - wikiPageKey: 'knowledge/global/pipeline.md', + wikiPageKey: 'wiki/global/pipeline.md', wikiScope: 'GLOBAL', wikiScopeId: null, refs: [{ connectionId: 'warehouse-2', sourceName: 'looker__b2b__sales_pipeline' }], @@ -1410,7 +1410,7 @@ describe('IngestBundleRunner — Stages 1 → 7', () => { connectionId: 'looker-run', targetConnectionId: null, artifactKind: 'wiki', - artifactKey: 'knowledge/global/pipeline.md', + artifactKey: 'wiki/global/pipeline.md', }), ]), ); @@ -1616,7 +1616,7 @@ describe('IngestBundleRunner — Stages 1 → 7', () => { const workUnitCall = deps.agentRunner.runLoop.mock.calls.find( ([params]: any[]) => params.telemetryTags.operationName === 'ingest-bundle-wu', ); - expect(workUnitCall?.[0].userPrompt).toContain('## Knowledge Pages'); + expect(workUnitCall?.[0].userPrompt).toContain('## Wiki Pages'); expect(workUnitCall?.[0].userPrompt).toContain( '- revenue-recognition: Recognize revenue net of refunds after fulfillment.', ); diff --git a/packages/context/src/ingest/ingest-bundle.runner.ts b/packages/context/src/ingest/ingest-bundle.runner.ts index d8f47c2a..582cbbf3 100644 --- a/packages/context/src/ingest/ingest-bundle.runner.ts +++ b/packages/context/src/ingest/ingest-bundle.runner.ts @@ -293,7 +293,7 @@ export class IngestBundleRunner { return '(empty)'; } - return `## Knowledge Pages\n${pages.map((page) => `- ${page.page_key}: ${page.summary}`).join('\n')}`; + return `## Wiki Pages\n${pages.map((page) => `- ${page.page_key}: ${page.summary}`).join('\n')}`; } private async buildSlIndex(connectionIds: string[]): Promise { @@ -596,7 +596,7 @@ export class IngestBundleRunner { const baseFraming = await this.deps.promptService.loadPrompt('memory_agent_bundle_ingest_work_unit'); const wuSkillNames = Array.from( - new Set([...adapter.skillNames, 'ingest_triage', 'sl_capture', 'knowledge_capture']), + new Set([...adapter.skillNames, 'ingest_triage', 'sl_capture', 'wiki_capture']), ); const wuSkills = await this.deps.skillsRegistry.listSkills(wuSkillNames, 'memory_agent'); const skillsPrompt = this.deps.skillsRegistry.buildSkillsPrompt(wuSkills, 'memory_agent'); @@ -973,7 +973,7 @@ export class IngestBundleRunner { const reconcileBaseFraming = await this.deps.promptService.loadPrompt('memory_agent_bundle_ingest_reconcile'); const reconcileSkills = await this.deps.skillsRegistry.listSkills( Array.from( - new Set(['ingest_triage', 'sl_capture', 'knowledge_capture', ...(adapter.reconcileSkillNames ?? [])]), + new Set(['ingest_triage', 'sl_capture', 'wiki_capture', ...(adapter.reconcileSkillNames ?? [])]), ), 'memory_agent', ); diff --git a/packages/context/src/ingest/ingest-runtime-assets.test.ts b/packages/context/src/ingest/ingest-runtime-assets.test.ts index 4b75fcdf..c77bee11 100644 --- a/packages/context/src/ingest/ingest-runtime-assets.test.ts +++ b/packages/context/src/ingest/ingest-runtime-assets.test.ts @@ -17,13 +17,13 @@ const adapterSkillNames = [ 'historic_sql_table_digest', 'historic_sql_patterns', 'ingest_triage', - 'knowledge_capture', + 'wiki_capture', 'sl_capture', ] as const; const adapterReconcileSkillNames = [ 'ingest_triage', - 'knowledge_capture', + 'wiki_capture', 'sl_capture', ] as const; diff --git a/packages/context/src/ingest/local-bundle-runtime.ts b/packages/context/src/ingest/local-bundle-runtime.ts index 9eeda894..2a3c9943 100644 --- a/packages/context/src/ingest/local-bundle-runtime.ts +++ b/packages/context/src/ingest/local-bundle-runtime.ts @@ -314,7 +314,7 @@ class LocalKnowledgeIndex implements KnowledgeIndexPort { scope: string, scopeId: string | null, ): Promise> { - const prefix = scope === 'GLOBAL' ? 'knowledge/global/' : `knowledge/user/${scopeId}/`; + const prefix = scope === 'GLOBAL' ? 'wiki/global/' : `wiki/user/${scopeId}/`; const result = new Map(); for (const [path, page] of this.sqlite.getExistingPages()) { if (!path.startsWith(prefix)) { @@ -341,7 +341,7 @@ class LocalKnowledgeIndex implements KnowledgeIndexPort { } async findPageByKey(scope: string, scopeId: string | null, pageKey: string) { - const path = scope === 'GLOBAL' ? `knowledge/global/${pageKey}.md` : `knowledge/user/${scopeId}/${pageKey}.md`; + const path = scope === 'GLOBAL' ? `wiki/global/${pageKey}.md` : `wiki/user/${scopeId}/${pageKey}.md`; try { await this.project.fileStore.readFile(path); return { page_key: pageKey }; @@ -355,12 +355,12 @@ class LocalKnowledgeIndex implements KnowledgeIndexPort { ): Promise { const pages: KnowledgeIndexPageListing[] = []; for (const scope of [ - { scope: 'GLOBAL', scopeId: null, dir: 'knowledge/global' }, - { scope: 'USER', scopeId: userId, dir: `knowledge/user/${userId}` }, + { scope: 'GLOBAL', scopeId: null, dir: 'wiki/global' }, + { scope: 'USER', scopeId: userId, dir: `wiki/user/${userId}` }, ]) { const listed = await this.project.fileStore.listFiles(scope.dir, true); for (const file of listed.files.filter((entry) => entry.endsWith('.md'))) { - const parsedPath = parseKnowledgeIndexPath(file.startsWith('global/') || file.startsWith('user/') ? file : `${scope.dir.replace('knowledge/', '')}/${file}`); + const parsedPath = parseKnowledgeIndexPath(file.startsWith('global/') || file.startsWith('user/') ? file : `${scope.dir.replace('wiki/', '')}/${file}`); if (!parsedPath || parsedPath.scope !== scope.scope) { continue; } @@ -404,7 +404,7 @@ class LocalKnowledgeIndex implements KnowledgeIndexPort { } private async syncAllPagesFromDisk(): Promise { - const listed = await this.project.fileStore.listFiles('knowledge', true); + const listed = await this.project.fileStore.listFiles('wiki', true); const existingPages = this.sqlite.getExistingPages(); const pages: SqliteKnowledgeIndexPage[] = []; for (const file of listed.files.filter((entry) => entry.endsWith('.md'))) { @@ -412,7 +412,7 @@ class LocalKnowledgeIndex implements KnowledgeIndexPort { if (!parsedPath) { continue; } - const path = `knowledge/${file}`; + const path = `wiki/${file}`; const raw = await this.project.fileStore.readFile(path); const parsed = parseWiki(raw.content); const tags = parseWikiTags(raw.content); diff --git a/packages/context/src/ingest/memory-flow/acceptance-fixtures.ts b/packages/context/src/ingest/memory-flow/acceptance-fixtures.ts index f4f01c12..66f1afb8 100644 --- a/packages/context/src/ingest/memory-flow/acceptance-fixtures.ts +++ b/packages/context/src/ingest/memory-flow/acceptance-fixtures.ts @@ -16,12 +16,12 @@ function baseScenario(overrides: Partial = {}): MemoryFlo { type: 'raw_snapshot_written', syncId: 'sync-success', rawFileCount: 4 }, { type: 'diff_computed', added: 2, modified: 1, deleted: 0, unchanged: 1 }, { type: 'chunks_planned', chunkCount: 2, workUnitCount: 2, evictionCount: 0 }, - { type: 'work_unit_started', unitKey: 'orders', skills: ['knowledge_capture'], stepBudget: 40 }, - { type: 'candidate_action', unitKey: 'orders', target: 'wiki', action: 'created', key: 'knowledge/global/orders.md' }, + { type: 'work_unit_started', unitKey: 'orders', skills: ['wiki_capture'], stepBudget: 40 }, + { type: 'candidate_action', unitKey: 'orders', target: 'wiki', action: 'created', key: 'wiki/global/orders.md' }, { type: 'candidate_action', unitKey: 'orders', target: 'sl', action: 'updated', key: 'warehouse.orders' }, { type: 'work_unit_finished', unitKey: 'orders', status: 'success' }, - { type: 'work_unit_started', unitKey: 'revenue', skills: ['knowledge_capture'], stepBudget: 40 }, - { type: 'candidate_action', unitKey: 'revenue', target: 'wiki', action: 'updated', key: 'knowledge/global/revenue.md' }, + { type: 'work_unit_started', unitKey: 'revenue', skills: ['wiki_capture'], stepBudget: 40 }, + { type: 'candidate_action', unitKey: 'revenue', target: 'wiki', action: 'updated', key: 'wiki/global/revenue.md' }, { type: 'work_unit_finished', unitKey: 'revenue', status: 'success' }, { type: 'reconciliation_finished', conflictCount: 0, fallbackCount: 0 }, { type: 'saved', commitSha: 'abc123456789', wikiCount: 2, slCount: 1 }, // pragma: allowlist secret @@ -38,7 +38,7 @@ function baseScenario(overrides: Partial = {}): MemoryFlo unitKey: 'orders', target: 'wiki', action: 'created', - key: 'knowledge/global/orders.md', + key: 'wiki/global/orders.md', summary: 'Captured order definitions', rawFiles: ['models/orders.yml'], status: 'success', @@ -56,7 +56,7 @@ function baseScenario(overrides: Partial = {}): MemoryFlo unitKey: 'revenue', target: 'wiki', action: 'updated', - key: 'knowledge/global/revenue.md', + key: 'wiki/global/revenue.md', summary: 'Updated revenue notes', rawFiles: ['docs/revenue.md'], status: 'success', @@ -66,7 +66,7 @@ function baseScenario(overrides: Partial = {}): MemoryFlo { rawPath: 'models/orders.yml', artifactKind: 'wiki', - artifactKey: 'knowledge/global/orders.md', + artifactKey: 'wiki/global/orders.md', actionType: 'created', }, { rawPath: 'models/orders.yml', artifactKind: 'sl', artifactKey: 'warehouse.orders', actionType: 'updated' }, @@ -111,7 +111,7 @@ export function validationRevertScenario(): MemoryFlowReplayInput { { type: 'raw_snapshot_written', syncId: 'sync-validation', rawFileCount: 1 }, { type: 'diff_computed', added: 1, modified: 0, deleted: 0, unchanged: 0 }, { type: 'chunks_planned', chunkCount: 1, workUnitCount: 1, evictionCount: 0 }, - { type: 'work_unit_started', unitKey: 'orders', skills: ['knowledge_capture'], stepBudget: 40 }, + { type: 'work_unit_started', unitKey: 'orders', skills: ['wiki_capture'], stepBudget: 40 }, { type: 'candidate_action', unitKey: 'orders', target: 'sl', action: 'updated', key: 'warehouse.orders' }, { type: 'work_unit_finished', diff --git a/packages/context/src/ingest/memory-flow/events.test.ts b/packages/context/src/ingest/memory-flow/events.test.ts index e65cfc83..be97342b 100644 --- a/packages/context/src/ingest/memory-flow/events.test.ts +++ b/packages/context/src/ingest/memory-flow/events.test.ts @@ -77,7 +77,7 @@ function reportSnapshot(): IngestReportSnapshot { { rawPath: 'views/orders.view.lkml', artifactKind: 'wiki', - artifactKey: 'knowledge/global/orders.md', + artifactKey: 'wiki/global/orders.md', actionType: 'wiki_written', }, { @@ -115,7 +115,7 @@ function reportSnapshot(): IngestReportSnapshot { rawFiles: ['views/orders.view.lkml'], status: 'success', actions: [ - { target: 'wiki', type: 'created', key: 'knowledge/global/orders.md', detail: 'order facts' }, + { target: 'wiki', type: 'created', key: 'wiki/global/orders.md', detail: 'order facts' }, { target: 'sl', type: 'updated', key: 'warehouse.orders', detail: 'order measures' }, ], touchedSlSources: [{ connectionId: 'warehouse', sourceName: 'warehouse.orders' }], @@ -180,7 +180,7 @@ describe('memory-flow event mapping', () => { unitKey: 'orders', target: 'wiki', action: 'created', - key: 'knowledge/global/orders.md', + key: 'wiki/global/orders.md', }); expect(replay.events).toContainEqual({ type: 'work_unit_finished', @@ -197,7 +197,7 @@ describe('memory-flow event mapping', () => { unitKey: 'orders', target: 'wiki', action: 'created', - key: 'knowledge/global/orders.md', + key: 'wiki/global/orders.md', summary: 'order facts', rawFiles: ['views/orders.view.lkml'], status: 'success', @@ -225,7 +225,7 @@ describe('memory-flow event mapping', () => { { rawPath: 'views/orders.view.lkml', artifactKind: 'wiki', - artifactKey: 'knowledge/global/orders.md', + artifactKey: 'wiki/global/orders.md', actionType: 'wiki_written', }, { diff --git a/packages/context/src/ingest/memory-flow/interaction.test.ts b/packages/context/src/ingest/memory-flow/interaction.test.ts index d997b236..290180df 100644 --- a/packages/context/src/ingest/memory-flow/interaction.test.ts +++ b/packages/context/src/ingest/memory-flow/interaction.test.ts @@ -43,7 +43,7 @@ function view(): MemoryFlowViewModel { unitKey: 'orders', target: 'wiki', action: 'created', - key: 'knowledge/orders.md', + key: 'wiki/orders.md', summary: 'order facts', rawFiles: ['orders.yml'], status: 'success', @@ -53,7 +53,7 @@ function view(): MemoryFlowViewModel { { rawPath: 'orders.yml', artifactKind: 'wiki', - artifactKey: 'knowledge/orders.md', + artifactKey: 'wiki/orders.md', actionType: 'wiki_written', }, ], @@ -104,8 +104,8 @@ function view(): MemoryFlowViewModel { status: 'complete', headline: '2 candidates', counters: ['1 wiki', '1 SL'], - chips: [{ label: 'knowledge/orders.md', status: 'complete' }], - details: ['wiki created: knowledge/orders.md', 'sl updated: warehouse.orders'], + chips: [{ label: 'wiki/orders.md', status: 'complete' }], + details: ['wiki created: wiki/orders.md', 'sl updated: warehouse.orders'], }, { id: 'gates', @@ -173,7 +173,7 @@ describe('memory-flow interaction reducer', () => { shouldQuit: false, }); expect(selectedMemoryFlowColumn(view(), selected).title).toBe('ACTIONS'); - expect(selectedMemoryFlowDetails(view(), selected)).toContain('wiki created: knowledge/orders.md'); + expect(selectedMemoryFlowDetails(view(), selected)).toContain('wiki created: wiki/orders.md'); }); it('selects and clamps a chip directly for mouse-driven renderers', () => { @@ -226,7 +226,7 @@ describe('memory-flow interaction reducer', () => { state = reduceMemoryFlowInteractionState(state, 'tab', view()); expect(state.pane).toBe('provenance'); expect(selectedMemoryFlowDetails(view(), state)).toContain( - 'orders.yml -> wiki:knowledge/orders.md (wiki_written)', + 'orders.yml -> wiki:wiki/orders.md (wiki_written)', ); state = reduceMemoryFlowInteractionState(state, 'tab', view()); @@ -241,7 +241,7 @@ describe('memory-flow interaction reducer', () => { state = reduceMemoryFlowInteractionState(state, 'provenance', view()); expect(state.pane).toBe('provenance'); expect(selectedMemoryFlowDetails(view(), state)).toContain( - 'orders.yml -> wiki:knowledge/orders.md (wiki_written)', + 'orders.yml -> wiki:wiki/orders.md (wiki_written)', ); state = reduceMemoryFlowInteractionState(state, 'transcript', view()); diff --git a/packages/context/src/ingest/memory-flow/interactive-render.test.ts b/packages/context/src/ingest/memory-flow/interactive-render.test.ts index a3ff0d5c..6b703a2a 100644 --- a/packages/context/src/ingest/memory-flow/interactive-render.test.ts +++ b/packages/context/src/ingest/memory-flow/interactive-render.test.ts @@ -36,7 +36,7 @@ function view(): MemoryFlowViewModel { unitKey: 'orders', target: 'wiki', action: 'created', - key: 'knowledge/orders.md', + key: 'wiki/orders.md', summary: 'order facts', rawFiles: ['orders.yml'], status: 'success', @@ -46,7 +46,7 @@ function view(): MemoryFlowViewModel { { rawPath: 'orders.yml', artifactKind: 'wiki', - artifactKey: 'knowledge/orders.md', + artifactKey: 'wiki/orders.md', actionType: 'wiki_written', }, ], @@ -97,8 +97,8 @@ function view(): MemoryFlowViewModel { status: 'complete', headline: '2 candidates', counters: ['1 wiki', '1 SL'], - chips: [{ label: 'knowledge/orders.md', status: 'complete' }], - details: ['wiki created: knowledge/orders.md', 'sl updated: warehouse.orders'], + chips: [{ label: 'wiki/orders.md', status: 'complete' }], + details: ['wiki created: wiki/orders.md', 'sl updated: warehouse.orders'], }, { id: 'gates', diff --git a/packages/context/src/ingest/memory-flow/render.test.ts b/packages/context/src/ingest/memory-flow/render.test.ts index e1bf425a..0053eefd 100644 --- a/packages/context/src/ingest/memory-flow/render.test.ts +++ b/packages/context/src/ingest/memory-flow/render.test.ts @@ -48,8 +48,8 @@ function view(): MemoryFlowViewModel { status: 'complete', headline: '2 candidates', counters: ['1 wiki', '1 SL'], - chips: [{ label: 'knowledge/orders.md', status: 'complete' }], - details: ['wiki created: knowledge/orders.md'], + chips: [{ label: 'wiki/orders.md', status: 'complete' }], + details: ['wiki created: wiki/orders.md'], }, { id: 'gates', diff --git a/packages/context/src/ingest/memory-flow/schema.test.ts b/packages/context/src/ingest/memory-flow/schema.test.ts index c1fbda64..c54752f8 100644 --- a/packages/context/src/ingest/memory-flow/schema.test.ts +++ b/packages/context/src/ingest/memory-flow/schema.test.ts @@ -21,9 +21,9 @@ function snapshot(overrides: Partial = {}): MemoryFlowRep { type: 'raw_snapshot_written', syncId: 'sync-1', rawFileCount: 2 }, { type: 'diff_computed', added: 1, modified: 1, deleted: 0, unchanged: 0 }, { type: 'chunks_planned', chunkCount: 1, workUnitCount: 1, evictionCount: 0 }, - { type: 'work_unit_started', unitKey: 'orders', skills: ['knowledge_capture'], stepBudget: 40 }, + { type: 'work_unit_started', unitKey: 'orders', skills: ['wiki_capture'], stepBudget: 40 }, { type: 'work_unit_step', unitKey: 'orders', stepIndex: 1, stepBudget: 40 }, - { type: 'candidate_action', unitKey: 'orders', target: 'wiki', action: 'created', key: 'knowledge/orders.md' }, + { type: 'candidate_action', unitKey: 'orders', target: 'wiki', action: 'created', key: 'wiki/orders.md' }, { type: 'work_unit_finished', unitKey: 'orders', status: 'success' }, { type: 'reconciliation_finished', conflictCount: 0, fallbackCount: 0 }, { type: 'saved', commitSha: 'abc12345', wikiCount: 1, slCount: 0 }, @@ -37,7 +37,7 @@ function snapshot(overrides: Partial = {}): MemoryFlowRep unitKey: 'orders', target: 'wiki', action: 'created', - key: 'knowledge/orders.md', + key: 'wiki/orders.md', summary: 'Created orders page', rawFiles: ['orders.md'], status: 'success', @@ -47,7 +47,7 @@ function snapshot(overrides: Partial = {}): MemoryFlowRep { rawPath: 'orders.md', artifactKind: 'wiki', - artifactKey: 'knowledge/orders.md', + artifactKey: 'wiki/orders.md', actionType: 'wiki_written', }, ], diff --git a/packages/context/src/ingest/memory-flow/view-model.test.ts b/packages/context/src/ingest/memory-flow/view-model.test.ts index 27322c69..4e6edae3 100644 --- a/packages/context/src/ingest/memory-flow/view-model.test.ts +++ b/packages/context/src/ingest/memory-flow/view-model.test.ts @@ -21,7 +21,7 @@ function replayInput(): MemoryFlowReplayInput { unitKey: 'orders', target: 'wiki', action: 'created', - key: 'knowledge/orders.md', + key: 'wiki/orders.md', summary: 'order facts', rawFiles: ['orders.yml'], status: 'success', @@ -40,7 +40,7 @@ function replayInput(): MemoryFlowReplayInput { { rawPath: 'orders.yml', artifactKind: 'wiki', - artifactKey: 'knowledge/orders.md', + artifactKey: 'wiki/orders.md', actionType: 'wiki_written', }, ], @@ -60,8 +60,8 @@ function replayInput(): MemoryFlowReplayInput { { type: 'raw_snapshot_written', syncId: 'sync-1', rawFileCount: 2 }, { type: 'diff_computed', added: 1, modified: 1, deleted: 0, unchanged: 3 }, { type: 'chunks_planned', chunkCount: 2, workUnitCount: 2, evictionCount: 0 }, - { type: 'work_unit_started', unitKey: 'orders', skills: ['knowledge_capture'], stepBudget: 40 }, - { type: 'candidate_action', unitKey: 'orders', target: 'wiki', action: 'created', key: 'knowledge/orders.md' }, + { type: 'work_unit_started', unitKey: 'orders', skills: ['wiki_capture'], stepBudget: 40 }, + { type: 'candidate_action', unitKey: 'orders', target: 'wiki', action: 'created', key: 'wiki/orders.md' }, { type: 'candidate_action', unitKey: 'orders', target: 'sl', action: 'updated', key: 'warehouse.orders' }, { type: 'work_unit_finished', unitKey: 'orders', status: 'success' }, { type: 'work_unit_finished', unitKey: 'revenue', status: 'failed', reason: 'validation failed' }, @@ -122,7 +122,7 @@ describe('buildMemoryFlowViewModel', () => { { rawPath: 'orders.yml', artifactKind: 'wiki', - artifactKey: 'knowledge/orders.md', + artifactKey: 'wiki/orders.md', actionType: 'wiki_written', }, ]); @@ -136,7 +136,7 @@ describe('buildMemoryFlowViewModel', () => { }, ]); expect(view.columns.find((column) => column.id === 'actions')?.details).toContain( - 'orders wiki created knowledge/orders.md: order facts', + 'orders wiki created wiki/orders.md: order facts', ); expect(view.columns.find((column) => column.id === 'saved')?.details).toContain('Commit: abc12345'); expect(view.completionLine).toBe( @@ -159,13 +159,13 @@ describe('buildMemoryFlowViewModel', () => { { type: 'source_acquired', adapter: 'looker', trigger: 'demo_seeded', fileCount: 7 }, { type: 'source_acquired', adapter: 'notion', trigger: 'demo_seeded', fileCount: 8 }, { type: 'chunks_planned', chunkCount: 1, workUnitCount: 1, evictionCount: 0 }, - { type: 'work_unit_started', unitKey: 'revenue-and-contracts', skills: ['knowledge_capture'], stepBudget: 40 }, + { type: 'work_unit_started', unitKey: 'revenue-and-contracts', skills: ['wiki_capture'], stepBudget: 40 }, { type: 'candidate_action', unitKey: 'revenue-and-contracts', target: 'wiki', action: 'created', - key: 'knowledge/global/arr-contract-first.md', + key: 'wiki/global/arr-contract-first.md', }, { type: 'work_unit_finished', unitKey: 'revenue-and-contracts', status: 'success' }, { type: 'reconciliation_finished', conflictCount: 0, fallbackCount: 0 }, @@ -376,7 +376,7 @@ describe('buildMemoryFlowViewModel', () => { { type: 'raw_snapshot_written', syncId: 'sync-errors', rawFileCount: 2 }, { type: 'diff_computed', added: 2, modified: 0, deleted: 0, unchanged: 0 }, { type: 'chunks_planned', chunkCount: 1, workUnitCount: 1, evictionCount: 0 }, - { type: 'work_unit_started', unitKey: 'orders', skills: ['knowledge_capture'], stepBudget: 40 }, + { type: 'work_unit_started', unitKey: 'orders', skills: ['wiki_capture'], stepBudget: 40 }, { type: 'candidate_action', unitKey: 'orders', target: 'sl', action: 'updated', key: 'warehouse.orders' }, { type: 'work_unit_finished', @@ -402,7 +402,7 @@ describe('buildMemoryFlowViewModel', () => { events: [ { type: 'source_acquired', adapter: 'metricflow', trigger: 'manual_resync', fileCount: 1 }, { type: 'chunks_planned', chunkCount: 1, workUnitCount: 1, evictionCount: 0 }, - { type: 'work_unit_started', unitKey: 'docs', skills: ['knowledge_capture'], stepBudget: 40 }, + { type: 'work_unit_started', unitKey: 'docs', skills: ['wiki_capture'], stepBudget: 40 }, { type: 'work_unit_finished', unitKey: 'docs', status: 'failed', reason: 'agent step budget exhausted' }, ], plannedWorkUnits: [{ unitKey: 'docs', rawFiles: ['docs.md'], peerFileCount: 0, dependencyCount: 0 }], diff --git a/packages/context/src/ingest/report-snapshot.test.ts b/packages/context/src/ingest/report-snapshot.test.ts index c949a3cc..bdf5b193 100644 --- a/packages/context/src/ingest/report-snapshot.test.ts +++ b/packages/context/src/ingest/report-snapshot.test.ts @@ -19,7 +19,7 @@ function validReportSnapshot() { rawFiles: ['cards/1.json', 'cards/2.json'], status: 'success', actions: [ - { target: 'wiki', type: 'created', key: 'knowledge/global/revenue.md', detail: 'Revenue overview' }, + { target: 'wiki', type: 'created', key: 'wiki/global/revenue.md', detail: 'Revenue overview' }, { target: 'sl', type: 'updated', key: 'warehouse.orders', detail: 'Added order amount measure' }, ], touchedSlSources: [{ connectionId: 'warehouse', sourceName: 'orders' }], @@ -38,7 +38,7 @@ function validReportSnapshot() { { rawPath: 'cards/1.json', artifactKind: 'wiki', - artifactKey: 'knowledge/global/revenue.md', + artifactKey: 'wiki/global/revenue.md', actionType: 'wiki_written', }, ], @@ -48,7 +48,7 @@ function validReportSnapshot() { path: 'tool-transcripts/cards.jsonl', toolCallCount: 3, errorCount: 0, - toolNames: ['knowledge_capture'], + toolNames: ['wiki_capture'], }, ], reconciliationActions: [], @@ -90,7 +90,7 @@ describe('parseIngestReportSnapshot', () => { { target: 'wiki', type: 'created', - key: 'knowledge/global/revenue.md', + key: 'wiki/global/revenue.md', detail: 'Revenue overview', targetConnectionId: null, }, diff --git a/packages/context/src/ingest/sqlite-bundle-ingest-store.test.ts b/packages/context/src/ingest/sqlite-bundle-ingest-store.test.ts index 2798c64a..cd6d2385 100644 --- a/packages/context/src/ingest/sqlite-bundle-ingest-store.test.ts +++ b/packages/context/src/ingest/sqlite-bundle-ingest-store.test.ts @@ -159,7 +159,7 @@ describe('SqliteBundleIngestStore', () => { rawPath: 'pages/revenue.md', rawContentHash: 'hash-old', artifactKind: 'wiki', - artifactKey: 'knowledge/global/revenue.md', + artifactKey: 'wiki/global/revenue.md', artifactContentHash: null, actionType: 'wiki_written', }, @@ -191,7 +191,7 @@ describe('SqliteBundleIngestStore', () => { rawPath: 'pages/revenue.md', rawContentHash: 'hash-new', artifactKind: 'wiki', - artifactKey: 'knowledge/global/revenue.md', + artifactKey: 'wiki/global/revenue.md', artifactContentHash: 'artifact-hash-new', actionType: 'wiki_written', }, @@ -234,7 +234,7 @@ describe('SqliteBundleIngestStore', () => { sync_id: 'sync-new', raw_content_hash: 'hash-new', artifact_kind: 'wiki', - artifact_key: 'knowledge/global/revenue.md', + artifact_key: 'wiki/global/revenue.md', action_type: 'wiki_written', }), expect.objectContaining({ @@ -381,7 +381,7 @@ describe('SqliteBundleIngestStore', () => { rawPath: 'pages/success/page.md', rawContentHash: 'hash-success', artifactKind: 'wiki', - artifactKey: 'knowledge/notion/success.md', + artifactKey: 'wiki/notion/success.md', artifactContentHash: 'artifact-success', actionType: 'wiki_written', }, diff --git a/packages/context/src/ingest/wiki-sl-ref-repair.ts b/packages/context/src/ingest/wiki-sl-ref-repair.ts index 7d3d48f3..e416c52b 100644 --- a/packages/context/src/ingest/wiki-sl-ref-repair.ts +++ b/packages/context/src/ingest/wiki-sl-ref-repair.ts @@ -91,7 +91,7 @@ export async function repairWikiSlRefs(input: { warnings: [...warnings, 'Skipped wiki sl_refs repair: config service cannot list wiki files.'], }; } - const listed = await listFiles('knowledge', true); + const listed = await listFiles('wiki', true); const repairs: WikiSlRefRepair[] = []; for (const file of listed.files.sort()) { diff --git a/packages/context/src/mcp/context-tools.ts b/packages/context/src/mcp/context-tools.ts index 48830d44..9f84b586 100644 --- a/packages/context/src/mcp/context-tools.ts +++ b/packages/context/src/mcp/context-tools.ts @@ -208,10 +208,10 @@ export function registerKtxContextTools(deps: RegisterKtxContextToolsDeps): void const knowledge = ports.knowledge; registerParsedTool( server, - 'knowledge_search', + 'wiki_search', { - title: 'Knowledge Search', - description: 'Search KTX knowledge pages and return ranked summaries.', + title: 'Wiki Search', + description: 'Search KTX wiki pages and return ranked summaries.', inputSchema: knowledgeSearchSchema.shape, }, knowledgeSearchSchema, @@ -227,25 +227,25 @@ export function registerKtxContextTools(deps: RegisterKtxContextToolsDeps): void registerParsedTool( server, - 'knowledge_read', + 'wiki_read', { - title: 'Knowledge Read', - description: 'Read a KTX knowledge page by key.', + title: 'Wiki Read', + description: 'Read a KTX wiki page by key.', inputSchema: knowledgeReadSchema.shape, }, knowledgeReadSchema, async (input) => { const page = await knowledge.read({ userId: userContext.userId, key: input.key }); - return page ? jsonToolResult(page) : jsonErrorToolResult(`Knowledge page "${input.key}" was not found.`); + return page ? jsonToolResult(page) : jsonErrorToolResult(`Wiki page "${input.key}" was not found.`); }, ); registerParsedTool( server, - 'knowledge_write', + 'wiki_write', { - title: 'Knowledge Write', - description: 'Create or replace a KTX knowledge page and its SL references.', + title: 'Wiki Write', + description: 'Create or replace a KTX wiki page and its SL references.', inputSchema: knowledgeWriteSchema.shape, }, knowledgeWriteSchema, diff --git a/packages/context/src/mcp/local-project-ports.test.ts b/packages/context/src/mcp/local-project-ports.test.ts index e3812960..b95e4ad1 100644 --- a/packages/context/src/mcp/local-project-ports.test.ts +++ b/packages/context/src/mcp/local-project-ports.test.ts @@ -341,7 +341,7 @@ describe('createLocalProjectMcpContextPorts', () => { }); }); - it('writes, reads, and searches global knowledge pages', async () => { + it('writes, reads, and searches global wiki pages', async () => { const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' }); const ports = createLocalProjectMcpContextPorts(project); @@ -372,7 +372,7 @@ describe('createLocalProjectMcpContextPorts', () => { results: [ expect.objectContaining({ key: 'revenue', - path: 'knowledge/global/revenue.md', + path: 'wiki/global/revenue.md', scope: 'GLOBAL', summary: 'Revenue definition', score: expect.any(Number), diff --git a/packages/context/src/mcp/server.test.ts b/packages/context/src/mcp/server.test.ts index 4430f6f7..193d8f67 100644 --- a/packages/context/src/mcp/server.test.ts +++ b/packages/context/src/mcp/server.test.ts @@ -76,7 +76,7 @@ describe('createKtxMcpServer', () => { captured: { wiki: ['revenue'], sl: [], xrefs: [] }, error: null, commitHash: 'abc123', - skillsLoaded: ['knowledge_capture'], + skillsLoaded: ['wiki_capture'], signalDetected: true, }), }; @@ -123,7 +123,7 @@ describe('createKtxMcpServer', () => { captured: { wiki: ['revenue'], sl: [], xrefs: [] }, error: null, commitHash: 'abc123', - skillsLoaded: ['knowledge_capture'], + skillsLoaded: ['wiki_capture'], signalDetected: true, }, null, @@ -139,7 +139,7 @@ describe('createKtxMcpServer', () => { captured: { wiki: ['revenue'], sl: [], xrefs: [] }, error: null, commitHash: 'abc123', - skillsLoaded: ['knowledge_capture'], + skillsLoaded: ['wiki_capture'], signalDetected: true, }, }); @@ -175,7 +175,7 @@ describe('createKtxMcpServer', () => { }: { toolSet: Record Promise }>; }) => { - await toolSet.load_skill.execute({ name: 'knowledge_capture' }); + await toolSet.load_skill.execute({ name: 'wiki_capture' }); await toolSet.wiki_write.execute( { key: 'arr', @@ -220,7 +220,7 @@ describe('createKtxMcpServer', () => { }); await expect(access(join(project.projectDir, '.ktx/db.sqlite'))).resolves.toBeUndefined(); await expect(access(join(project.projectDir, '.ktx/memory-runs/memory-run-mcp.json'))).rejects.toThrow(); - await expect(readFile(join(project.projectDir, 'knowledge/global/arr.md'), 'utf-8')).resolves.toContain( + await expect(readFile(join(project.projectDir, 'wiki/global/arr.md'), 'utf-8')).resolves.toContain( 'ARR means annual recurring revenue.', ); } finally { @@ -257,7 +257,7 @@ describe('createKtxMcpServer', () => { results: [ { key: 'revenue', - path: 'knowledge/global/revenue.md', + path: 'wiki/global/revenue.md', scope: 'GLOBAL', summary: 'Paid order value', score: 0.42, @@ -519,9 +519,6 @@ describe('createKtxMcpServer', () => { 'ingest_report', 'ingest_status', 'ingest_trigger', - 'knowledge_read', - 'knowledge_search', - 'knowledge_write', 'memory_capture', 'memory_capture_status', 'scan_list_artifacts', @@ -534,6 +531,9 @@ describe('createKtxMcpServer', () => { 'sl_read_source', 'sl_validate', 'sl_write_source', + 'wiki_read', + 'wiki_search', + 'wiki_write', ]); await expect(getTool(fake.tools, 'connection_list').handler({})).resolves.toEqual({ @@ -595,20 +595,20 @@ describe('createKtxMcpServer', () => { }); expect(contextTools.connections?.test).toHaveBeenCalledWith({ connectionId: 'warehouse' }); - await getTool(fake.tools, 'knowledge_search').handler({ query: 'revenue', limit: 5 }); + await getTool(fake.tools, 'wiki_search').handler({ query: 'revenue', limit: 5 }); expect(contextTools.knowledge?.search).toHaveBeenCalledWith({ userId: 'mcp-user', query: 'revenue', limit: 5, }); - await getTool(fake.tools, 'knowledge_read').handler({ key: 'revenue' }); + await getTool(fake.tools, 'wiki_read').handler({ key: 'revenue' }); expect(contextTools.knowledge?.read).toHaveBeenCalledWith({ userId: 'mcp-user', key: 'revenue', }); - await getTool(fake.tools, 'knowledge_write').handler({ + await getTool(fake.tools, 'wiki_write').handler({ key: 'revenue', summary: 'Paid order value', content: '# Revenue', diff --git a/packages/context/src/memory/capture-signals.ts b/packages/context/src/memory/capture-signals.ts index 856df30b..360f0b7c 100644 --- a/packages/context/src/memory/capture-signals.ts +++ b/packages/context/src/memory/capture-signals.ts @@ -9,7 +9,7 @@ const LOOKML_STRUCTURAL_PATTERN = /^\s*(view|explore|model|include)\s*:\s*[\w"`] const LOOKML_FIELDS_PATTERN = /^\s*(measure|dimension|dimension_group|sql_table_name|derived_table|sql_always_where|drill_fields|join)\s*:/m; -export const DEFAULT_SKILL_NAMES = ['sl', 'sl_capture', 'knowledge_capture'] as const; +export const DEFAULT_SKILL_NAMES = ['sl', 'sl_capture', 'wiki_capture'] as const; export function detectCaptureSignals(input: MemoryAgentInput): CaptureSignals { const userMessage = input.userMessage?.trim() ?? ''; @@ -56,7 +56,7 @@ export function buildRequiredSkillsBlock(signals: CaptureSignals): string { const reason = signals.reasons.find((r) => r.includes('definition keyword') || r.includes('definition table')) ?? 'wiki signal detected'; - required.push({ name: 'knowledge_capture', reason }); + required.push({ name: 'wiki_capture', reason }); } if (signals.sl) { const reason = diff --git a/packages/context/src/memory/local-memory.test.ts b/packages/context/src/memory/local-memory.test.ts index 1284f76d..e44a5bf1 100644 --- a/packages/context/src/memory/local-memory.test.ts +++ b/packages/context/src/memory/local-memory.test.ts @@ -40,7 +40,7 @@ describe('LocalMemoryRunStore', () => { await store.markDone('memory-run-1', { signalDetected: true, actions: [{ target: 'wiki', type: 'created', key: 'revenue', detail: 'Revenue definition' }], - skillsLoaded: ['knowledge_capture'], + skillsLoaded: ['wiki_capture'], commitHash: 'abc123', }); @@ -69,7 +69,7 @@ describe('LocalMemoryRunStore', () => { chatId: 'chat-1', outputSummary: { actions: [{ target: 'wiki', type: 'created', key: 'revenue', detail: 'Revenue definition' }], - skillsLoaded: ['knowledge_capture'], + skillsLoaded: ['wiki_capture'], signalDetected: true, commitHash: 'abc123', }, @@ -96,7 +96,7 @@ describe('createLocalProjectMemoryCapture', () => { }: { toolSet: Record Promise }>; }) => { - await toolSet.load_skill.execute({ name: 'knowledge_capture' }); + await toolSet.load_skill.execute({ name: 'wiki_capture' }); await toolSet.wiki_write.execute( { key: 'revenue', @@ -134,11 +134,11 @@ describe('createLocalProjectMemoryCapture', () => { status: 'done', done: true, captured: { wiki: ['revenue'], sl: [], xrefs: [] }, - skillsLoaded: ['knowledge_capture'], + skillsLoaded: ['wiki_capture'], signalDetected: true, }); - await expect(readFile(join(project.projectDir, 'knowledge/global/revenue.md'), 'utf-8')).resolves.toContain( + await expect(readFile(join(project.projectDir, 'wiki/global/revenue.md'), 'utf-8')).resolves.toContain( 'Revenue means paid order value net of refunds.', ); }); diff --git a/packages/context/src/memory/local-memory.ts b/packages/context/src/memory/local-memory.ts index af65b54e..3cc9d324 100644 --- a/packages/context/src/memory/local-memory.ts +++ b/packages/context/src/memory/local-memory.ts @@ -222,8 +222,8 @@ class LocalKnowledgeIndex implements KnowledgeIndexPort { async listPagesForUser(userId: string) { const pages: KnowledgeIndexPageListing[] = []; for (const scope of [ - { scope: 'GLOBAL', scopeId: null, dir: 'knowledge/global' }, - { scope: 'USER', scopeId: userId, dir: `knowledge/user/${userId}` }, + { scope: 'GLOBAL', scopeId: null, dir: 'wiki/global' }, + { scope: 'USER', scopeId: userId, dir: `wiki/user/${userId}` }, ]) { const listed = await this.project.fileStore.listFiles(scope.dir, true); for (const file of listed.files.filter((entry) => entry.endsWith('.md'))) { @@ -262,7 +262,7 @@ class LocalKnowledgeIndex implements KnowledgeIndexPort { } private pagePath(scope: string, scopeId: string | null, pageKey: string): string { - return scope === 'GLOBAL' ? `knowledge/global/${pageKey}.md` : `knowledge/user/${scopeId}/${pageKey}.md`; + return scope === 'GLOBAL' ? `wiki/global/${pageKey}.md` : `wiki/user/${scopeId}/${pageKey}.md`; } } diff --git a/packages/context/src/memory/memory-agent.service.ts b/packages/context/src/memory/memory-agent.service.ts index 6f239053..437111e4 100644 --- a/packages/context/src/memory/memory-agent.service.ts +++ b/packages/context/src/memory/memory-agent.service.ts @@ -318,7 +318,7 @@ export class MemoryAgentService { } const signalsActedOn: string[] = []; - if (signals.knowledge && skillsLoaded.includes('knowledge_capture')) { + if (signals.knowledge && skillsLoaded.includes('wiki_capture')) { signalsActedOn.push('knowledge'); } if (signals.sl && skillsLoaded.includes('sl')) { @@ -580,12 +580,12 @@ export class MemoryAgentService { private async buildWikiIndex(userId: string, userScopedEnabled: boolean): Promise { const pages = await this.deps.knowledgeIndex.listPagesForUser(userId); if (pages.length === 0) { - return '(empty — no knowledge pages exist yet)'; + return '(empty — no wiki pages exist yet)'; } const formatEntry = (p: { page_key: string; summary: string }) => `- ${p.page_key}: ${p.summary}`; if (!userScopedEnabled) { - return `## Knowledge Pages\n${pages.map(formatEntry).join('\n')}`; + return `## Wiki Pages\n${pages.map(formatEntry).join('\n')}`; } const globalEntries: string[] = []; diff --git a/packages/context/src/memory/memory-runs.test.ts b/packages/context/src/memory/memory-runs.test.ts index 75c25a38..049936ad 100644 --- a/packages/context/src/memory/memory-runs.test.ts +++ b/packages/context/src/memory/memory-runs.test.ts @@ -96,7 +96,7 @@ describe('MemoryCaptureService', () => { const result: MemoryAgentResult = { signalDetected: true, actions: [{ target: 'wiki', type: 'created', key: 'revenue', detail: 'captured revenue definition' }], - skillsLoaded: ['knowledge_capture'], + skillsLoaded: ['wiki_capture'], commitHash: 'abc123', }; const { capture, store, ingest, run } = buildService(); @@ -136,7 +136,7 @@ describe('MemoryCaptureService', () => { }, error: null, commitHash: 'abc123', - skillsLoaded: ['knowledge_capture'], + skillsLoaded: ['wiki_capture'], signalDetected: true, }); expect(store.rows.get('run-1')?.inputHash).toHaveLength(64); diff --git a/packages/context/src/memory/memory-runtime-assets.test.ts b/packages/context/src/memory/memory-runtime-assets.test.ts index bd18e524..973d7271 100644 --- a/packages/context/src/memory/memory-runtime-assets.test.ts +++ b/packages/context/src/memory/memory-runtime-assets.test.ts @@ -10,7 +10,7 @@ const promptsDir = fileURLToPath(new URL('../../prompts', import.meta.url)); const skillsDir = fileURLToPath(new URL('../../skills', import.meta.url)); const memorySourceTypes: MemoryAgentSourceType[] = ['research', 'external_ingest', 'backfill']; const expectedSkillHeadings: Record = { - knowledge_capture: '# Knowledge Capture', + wiki_capture: '# Wiki Capture', sl: '# Semantic Layer', sl_capture: '# Semantic Layer', }; @@ -33,7 +33,7 @@ const verificationWriterSkills = [ 'live_database_ingest', 'historic_sql_table_digest', 'historic_sql_patterns', - 'knowledge_capture', + 'wiki_capture', 'sl_capture', ] as const; @@ -77,7 +77,7 @@ describe('memory runtime assets', () => { const registry = new SkillsRegistryService({ skillsDir }); const skills = await registry.listSkills([...DEFAULT_SKILL_NAMES], 'memory_agent'); - expect(skills.map((skill) => skill.name).sort()).toEqual(['knowledge_capture', 'sl', 'sl_capture']); + expect(skills.map((skill) => skill.name).sort()).toEqual(['sl', 'sl_capture', 'wiki_capture']); for (const skill of skills) { const body = await readFile(join(skill.path, 'SKILL.md'), 'utf-8'); diff --git a/packages/context/src/project/config.test.ts b/packages/context/src/project/config.test.ts index 5f13729e..ee6b8ee9 100644 --- a/packages/context/src/project/config.test.ts +++ b/packages/context/src/project/config.test.ts @@ -37,7 +37,7 @@ describe('KTX project config', () => { run_research: { enabled: false, max_iterations: 20, - default_toolset: ['sl_query', 'knowledge_search', 'sl_read_source'], + default_toolset: ['sl_query', 'wiki_search', 'sl_read_source'], }, }, memory: { diff --git a/packages/context/src/project/config.ts b/packages/context/src/project/config.ts index b00d0148..ad5ecb8a 100644 --- a/packages/context/src/project/config.ts +++ b/packages/context/src/project/config.ts @@ -408,7 +408,7 @@ export function buildDefaultKtxProjectConfig(projectName = 'ktx-project'): KtxPr run_research: { enabled: false, max_iterations: 20, - default_toolset: ['sl_query', 'knowledge_search', 'sl_read_source'], + default_toolset: ['sl_query', 'wiki_search', 'sl_read_source'], }, }, memory: { diff --git a/packages/context/src/project/local-git-file-store.test.ts b/packages/context/src/project/local-git-file-store.test.ts index 94085488..62b7fc8c 100644 --- a/packages/context/src/project/local-git-file-store.test.ts +++ b/packages/context/src/project/local-git-file-store.test.ts @@ -32,7 +32,7 @@ describe('LocalGitFileStore', () => { it('writes, commits, and reads a project file', async () => { const write = await store.writeFile( - 'knowledge/global/revenue.md', + 'wiki/global/revenue.md', '# Revenue\n', 'Agent', 'agent@example.com', @@ -40,20 +40,20 @@ describe('LocalGitFileStore', () => { ); expect(write.commitHash).toMatch(/^[0-9a-f]{40}$/); - await expect(readFile(join(tempDir, 'knowledge/global/revenue.md'), 'utf-8')).resolves.toBe('# Revenue\n'); - await expect(store.readFile('knowledge/global/revenue.md')).resolves.toMatchObject({ + await expect(readFile(join(tempDir, 'wiki/global/revenue.md'), 'utf-8')).resolves.toBe('# Revenue\n'); + await expect(store.readFile('wiki/global/revenue.md')).resolves.toMatchObject({ content: '# Revenue\n', }); }); it('lists files recursively and can strip the requested prefix', async () => { - await store.writeFile('knowledge/global/a.md', 'a', 'Agent', 'agent@example.com', 'Add a'); - await store.writeFile('knowledge/global/nested/b.md', 'b', 'Agent', 'agent@example.com', 'Add b'); + await store.writeFile('wiki/global/a.md', 'a', 'Agent', 'agent@example.com', 'Add a'); + await store.writeFile('wiki/global/nested/b.md', 'b', 'Agent', 'agent@example.com', 'Add b'); - await expect(store.listFiles('knowledge')).resolves.toEqual({ - files: ['knowledge/global/a.md', 'knowledge/global/nested/b.md'], + await expect(store.listFiles('wiki')).resolves.toEqual({ + files: ['wiki/global/a.md', 'wiki/global/nested/b.md'], }); - await expect(store.listFiles('knowledge/global', true)).resolves.toEqual({ + await expect(store.listFiles('wiki/global', true)).resolves.toEqual({ files: ['a.md', 'nested/b.md'], }); }); @@ -77,10 +77,10 @@ describe('LocalGitFileStore', () => { }); it('exposes Git history for a file', async () => { - await store.writeFile('knowledge/global/history.md', 'v1', 'Agent', 'agent@example.com', 'Add history'); - await store.writeFile('knowledge/global/history.md', 'v2', 'Agent', 'agent@example.com', 'Update history'); + await store.writeFile('wiki/global/history.md', 'v1', 'Agent', 'agent@example.com', 'Add history'); + await store.writeFile('wiki/global/history.md', 'v2', 'Agent', 'agent@example.com', 'Update history'); - const history = await store.getFileHistory('knowledge/global/history.md'); + const history = await store.getFileHistory('wiki/global/history.md'); expect(Array.isArray(history)).toBe(true); expect(history[0]).toMatchObject({ message: 'Update history' }); diff --git a/packages/context/src/project/project.test.ts b/packages/context/src/project/project.test.ts index b6e88604..caf36220 100644 --- a/packages/context/src/project/project.test.ts +++ b/packages/context/src/project/project.test.ts @@ -37,7 +37,7 @@ describe('KTX local project runtime', () => { expect(gitignore).toContain('secrets/'); expect(gitignore).toContain('setup/'); expect(gitignore).toContain('agents/'); - await expect(stat(join(projectDir, 'knowledge/global/.gitkeep'))).resolves.toBeDefined(); + await expect(stat(join(projectDir, 'wiki/global/.gitkeep'))).resolves.toBeDefined(); await expect(stat(join(projectDir, 'semantic-layer/.gitkeep'))).resolves.toBeDefined(); await expect(stat(join(projectDir, '_schema/.gitkeep'))).rejects.toMatchObject({ code: 'ENOENT' }); await expect(stat(join(projectDir, 'raw-sources/.gitkeep'))).resolves.toBeDefined(); @@ -50,7 +50,7 @@ describe('KTX local project runtime', () => { const loaded = await loadKtxProject({ projectDir }); await loaded.fileStore.writeFile( - 'knowledge/global/revenue.md', + 'wiki/global/revenue.md', '# Revenue\n', 'Agent', 'agent@example.com', @@ -58,7 +58,7 @@ describe('KTX local project runtime', () => { ); expect(loaded.config.project).toBe('warehouse'); - await expect(loaded.fileStore.readFile('knowledge/global/revenue.md')).resolves.toMatchObject({ + await expect(loaded.fileStore.readFile('wiki/global/revenue.md')).resolves.toMatchObject({ content: '# Revenue\n', }); }); diff --git a/packages/context/src/project/project.ts b/packages/context/src/project/project.ts index 59e594a2..50f89262 100644 --- a/packages/context/src/project/project.ts +++ b/packages/context/src/project/project.ts @@ -41,7 +41,7 @@ const TRACKED_SCAFFOLD_FILES: Array<{ path: string; content: string }> = [ }, { path: '.ktx/prompts/.gitkeep', content: '' }, { path: '.ktx/skills/.gitkeep', content: '' }, - { path: 'knowledge/global/.gitkeep', content: '' }, + { path: 'wiki/global/.gitkeep', content: '' }, { path: 'semantic-layer/.gitkeep', content: '' }, { path: 'raw-sources/.gitkeep', content: '' }, ]; diff --git a/packages/context/src/skills/skills-registry.service.test.ts b/packages/context/src/skills/skills-registry.service.test.ts index 82c7c8ab..9bb716dd 100644 --- a/packages/context/src/skills/skills-registry.service.test.ts +++ b/packages/context/src/skills/skills-registry.service.test.ts @@ -64,14 +64,14 @@ describe('SkillsRegistryService', () => { it('discovers valid skills and skips invalid ones', async () => { await writeSkill('sl', '---\nname: sl\ndescription: Semantic layer.\n---\n\n# SL'); - await writeSkill('knowledge_capture', '---\nname: knowledge_capture\ndescription: Wiki capture.\n---\n\n# KC'); + await writeSkill('wiki_capture', '---\nname: wiki_capture\ndescription: Wiki capture.\n---\n\n# KC'); await writeSkill('broken', '# no frontmatter at all'); await mkdir(join(tempDir, 'not_a_skill'), { recursive: true }); const catalog = await service.discoverSkills(tempDir); expect(catalog.size).toBe(2); expect(catalog.get('sl')?.name).toBe('sl'); - expect(catalog.get('knowledge_capture')?.description).toContain('Wiki capture'); + expect(catalog.get('wiki_capture')?.description).toContain('Wiki capture'); expect(catalog.has('broken')).toBe(false); }); }); @@ -80,10 +80,10 @@ describe('SkillsRegistryService', () => { it('formats bullet list with name and description', () => { const output = service.buildSkillsPrompt([ { name: 'sl', description: 'Semantic layer.', path: '/tmp/sl' }, - { name: 'knowledge_capture', description: 'Wiki capture.', path: '/tmp/kc' }, + { name: 'wiki_capture', description: 'Wiki capture.', path: '/tmp/kc' }, ]); expect(output).toContain('- sl: Semantic layer.'); - expect(output).toContain('- knowledge_capture: Wiki capture.'); + expect(output).toContain('- wiki_capture: Wiki capture.'); expect(output).toContain('Use the `load_skill` tool'); }); @@ -144,8 +144,8 @@ describe('SkillsRegistryService', () => { '---\nname: sl_capture\ndescription: Memory-only capture skill.\ncallers: [memory_agent]\n---\n\n# Capture', ); await writeSkill( - 'knowledge_capture', - '---\nname: knowledge_capture\ndescription: Wiki capture.\ncallers: [memory_agent]\n---\n\n# KC', + 'wiki_capture', + '---\nname: wiki_capture\ndescription: Wiki capture.\ncallers: [memory_agent]\n---\n\n# KC', ); service = new SkillsRegistryService({ skillsDir: tempDir }); }); @@ -157,7 +157,7 @@ describe('SkillsRegistryService', () => { it('memory_agent caller sees memory-only and open skills', async () => { const skills = await service.listSkills('memory_agent'); - expect(skills.map((skill) => skill.name).sort()).toEqual(['knowledge_capture', 'sl', 'sl_capture']); + expect(skills.map((skill) => skill.name).sort()).toEqual(['sl', 'sl_capture', 'wiki_capture']); }); it('listSkills with names and caller intersects both filters', async () => { @@ -185,26 +185,26 @@ describe('SkillsRegistryService', () => { it('discovers skills from additional directories when the primary directory misses', async () => { const extraDir = await mkdtemp(join(tmpdir(), 'skills-registry-extra-')); try { - await mkdir(join(extraDir, 'knowledge_capture'), { recursive: true }); + await mkdir(join(extraDir, 'wiki_capture'), { recursive: true }); await writeFile( - join(extraDir, 'knowledge_capture', 'SKILL.md'), + join(extraDir, 'wiki_capture', 'SKILL.md'), [ '---', - 'name: knowledge_capture', + 'name: wiki_capture', 'description: Packaged knowledge capture skill.', 'callers: [memory_agent]', '---', '', - '# Knowledge Capture', + '# Wiki Capture', ].join('\n'), 'utf-8', ); service = new SkillsRegistryService({ skillsDir: tempDir, additionalSkillDirs: [extraDir] }); - const skills = await service.listSkills(['knowledge_capture'], 'memory_agent'); + const skills = await service.listSkills(['wiki_capture'], 'memory_agent'); - expect(skills.map((skill) => skill.name)).toEqual(['knowledge_capture']); - expect(skills[0]?.path).toBe(join(extraDir, 'knowledge_capture')); + expect(skills.map((skill) => skill.name)).toEqual(['wiki_capture']); + expect(skills[0]?.path).toBe(join(extraDir, 'wiki_capture')); } finally { await rm(extraDir, { recursive: true, force: true }); } diff --git a/packages/context/src/skills/skills-registry.service.ts b/packages/context/src/skills/skills-registry.service.ts index 2f0e8de2..cd33e6d8 100644 --- a/packages/context/src/skills/skills-registry.service.ts +++ b/packages/context/src/skills/skills-registry.service.ts @@ -223,7 +223,7 @@ export class SkillsRegistryService { const list = skills.map((skill) => `- ${skill.name}: ${skill.description}`).join('\n'); const captureNote = caller === 'research' - ? '\n\nKnowledge pages and semantic-layer sources are captured automatically by a post-turn memory agent. Focus on answering, not on saving. Use `knowledge_read`/`knowledge_search` and `sl_read_source` to consult what already exists; the memory agent will write any new conventions or measures the turn surfaces.' + ? '\n\nWiki pages and semantic-layer sources are captured automatically by a post-turn memory agent. Focus on answering, not on saving. Use `wiki_read`/`wiki_search` and `sl_read_source` to consult what already exists; the memory agent will write any new conventions or measures the turn surfaces.' : ''; return `\n## Skills\n\nUse the \`load_skill\` tool to load a skill when the task benefits from specialized instructions.${captureNote}\n\nAvailable skills:\n${list}\n`; } diff --git a/packages/context/src/wiki/knowledge-wiki.service.test.ts b/packages/context/src/wiki/knowledge-wiki.service.test.ts index 40056edc..f7bb86e4 100644 --- a/packages/context/src/wiki/knowledge-wiki.service.test.ts +++ b/packages/context/src/wiki/knowledge-wiki.service.test.ts @@ -84,9 +84,9 @@ describe('KnowledgeWikiService.syncFromCommit', () => { const { service, pagesRepository, gitService } = makeService(); gitService.diffNameStatus.mockResolvedValue([ - { status: 'A', path: 'knowledge/global/new-page.md' }, - { status: 'M', path: 'knowledge/global/changed-page.md' }, - { status: 'D', path: 'knowledge/global/gone-page.md' }, + { status: 'A', path: 'wiki/global/new-page.md' }, + { status: 'M', path: 'wiki/global/changed-page.md' }, + { status: 'D', path: 'wiki/global/gone-page.md' }, ]); gitService.getFileAtCommit.mockImplementation((path: string) => { if (path.endsWith('new-page.md')) { @@ -117,10 +117,10 @@ describe('KnowledgeWikiService.syncFromCommit', () => { const { service, pagesRepository, gitService, logger } = makeService(); gitService.diffNameStatus.mockResolvedValue([ - { status: 'A', path: 'knowledge/global/revenue-policy.md' }, - { status: 'A', path: 'knowledge/global/historic-sql-order-lifecycle.md' }, - { status: 'A', path: 'knowledge/global/historic-sql/order-lifecycle.md' }, - { status: 'A', path: 'knowledge/global/orbit/company-overview.md' }, + { status: 'A', path: 'wiki/global/revenue-policy.md' }, + { status: 'A', path: 'wiki/global/historic-sql-order-lifecycle.md' }, + { status: 'A', path: 'wiki/global/historic-sql/order-lifecycle.md' }, + { status: 'A', path: 'wiki/global/orbit/company-overview.md' }, ]); gitService.getFileAtCommit.mockImplementation((path: string) => { if (path.endsWith('revenue-policy.md')) { @@ -137,13 +137,13 @@ describe('KnowledgeWikiService.syncFromCommit', () => { await service.syncFromCommit('sha-before', 'sha-after', 'run-uuid'); - expect(gitService.getFileAtCommit).not.toHaveBeenCalledWith('knowledge/global/orbit/company-overview.md', 'sha-after'); - expect(gitService.getFileAtCommit).not.toHaveBeenCalledWith('knowledge/global/historic-sql/order-lifecycle.md', 'sha-after'); + expect(gitService.getFileAtCommit).not.toHaveBeenCalledWith('wiki/global/orbit/company-overview.md', 'sha-after'); + expect(gitService.getFileAtCommit).not.toHaveBeenCalledWith('wiki/global/historic-sql/order-lifecycle.md', 'sha-after'); expect(logger.warn).toHaveBeenCalledWith( - '[knowledge.sync] skipping unparseable path: knowledge/global/orbit/company-overview.md', + '[wiki.sync] skipping unparseable path: wiki/global/orbit/company-overview.md', ); expect(logger.warn).toHaveBeenCalledWith( - '[knowledge.sync] skipping unparseable path: knowledge/global/historic-sql/order-lifecycle.md', + '[wiki.sync] skipping unparseable path: wiki/global/historic-sql/order-lifecycle.md', ); const call = pagesRepository.applyDiffTransactional.mock.calls[0][0]; expect(call.upserts).toEqual( diff --git a/packages/context/src/wiki/knowledge-wiki.service.ts b/packages/context/src/wiki/knowledge-wiki.service.ts index fb152e83..c8e276ab 100644 --- a/packages/context/src/wiki/knowledge-wiki.service.ts +++ b/packages/context/src/wiki/knowledge-wiki.service.ts @@ -7,7 +7,7 @@ import { buildKnowledgeSearchText } from './knowledge-search-text.js'; import type { KnowledgeGitDiffPort, KnowledgeIndexPort, UpsertPageParams } from './ports.js'; import type { WikiFrontmatter, WikiPage, WikiPageWithScope } from './types.js'; -const WIKI_PREFIX = 'knowledge'; +const WIKI_PREFIX = 'wiki'; export type { WikiFrontmatter }; @@ -89,7 +89,7 @@ export class KnowledgeWikiService { ) { const path = this.pagePath(scope, scopeId, pageKey); const serialized = this.serializePage(frontmatter, content); - const message = commitMessage ?? `Update knowledge page: ${pageKey}`; + const message = commitMessage ?? `Update wiki page: ${pageKey}`; return this.configService.writeFile(path, serialized, author, authorEmail, message, { skipLock: options?.skipLock, }); @@ -115,7 +115,7 @@ export class KnowledgeWikiService { ) { const path = this.pagePath(scope, scopeId, pageKey); try { - return await this.configService.deleteFile(path, author, authorEmail, `Remove knowledge page: ${pageKey}`); + return await this.configService.deleteFile(path, author, authorEmail, `Remove wiki page: ${pageKey}`); } catch (error) { // Check if the file actually exists — if not, deletion is a no-op try { @@ -196,7 +196,7 @@ export class KnowledgeWikiService { rawContent, author, authorEmail, - commitMessage ?? `Update knowledge page (raw): ${pageKey}`, + commitMessage ?? `Update wiki page (raw): ${pageKey}`, ); await this.syncSinglePage(scope, scopeId, pageKey, parsed.frontmatter, parsed.content); return parsed; @@ -352,9 +352,9 @@ export class KnowledgeWikiService { /** * Apply the diff between two commits on the config repo to the shared - * `knowledge` index in a single transaction. Called by the ingest runner + * wiki index in a single transaction. Called by the ingest runner * after Stage 6 squashes the session branch into main: the pre-squash main - * SHA and the post-squash SHA bracket exactly the set of knowledge-file + * SHA and the post-squash SHA bracket exactly the set of wiki-file * changes this run produced. * * Any added/modified file becomes an upsert (tagged with `source_run_id`), @@ -362,7 +362,7 @@ export class KnowledgeWikiService { * transaction so the shared table stays consistent. */ async syncFromCommit(fromSha: string, toSha: string, runId: string): Promise { - const diff = await this.gitService.diffNameStatus(fromSha, toSha, 'knowledge/'); + const diff = await this.gitService.diffNameStatus(fromSha, toSha, 'wiki/'); if (diff.length === 0) { return; } @@ -372,7 +372,7 @@ export class KnowledgeWikiService { for (const entry of diff) { const parsedPath = parseKnowledgePath(entry.path); if (!parsedPath) { - this.logger.warn(`[knowledge.sync] skipping unparseable path: ${entry.path}`); + this.logger.warn(`[wiki.sync] skipping unparseable path: ${entry.path}`); continue; } if (entry.status === 'D') { @@ -392,7 +392,7 @@ export class KnowledgeWikiService { embedding = await this.embeddingService.computeEmbedding(searchText); } catch (err) { this.logger.warn( - `[knowledge.sync] embedding failed for ${parsedPath.pageKey}: ${err instanceof Error ? err.message : String(err)}`, + `[wiki.sync] embedding failed for ${parsedPath.pageKey}: ${err instanceof Error ? err.message : String(err)}`, ); } const contentHash = createHash('sha256').update(content).digest('hex'); @@ -410,21 +410,21 @@ export class KnowledgeWikiService { } await this.pagesRepository.applyDiffTransactional({ runId, upserts, deletes }); - this.logger.log(`[knowledge.sync] run=${runId} applied ${upserts.length} upsert(s), ${deletes.length} delete(s)`); + this.logger.log(`[wiki.sync] run=${runId} applied ${upserts.length} upsert(s), ${deletes.length} delete(s)`); } } /** - * Parse a `knowledge//...` file path into its scope and page key. - * `knowledge/global/foo.md` → { scope: 'GLOBAL', scopeId: null, pageKey: 'foo' } - * `knowledge/user//bar.md` → { scope: 'USER', scopeId: '', pageKey: 'bar' } + * Parse a `wiki//...` file path into its scope and page key. + * `wiki/global/foo.md` → { scope: 'GLOBAL', scopeId: null, pageKey: 'foo' } + * `wiki/user//bar.md` → { scope: 'USER', scopeId: '', pageKey: 'bar' } */ function parseKnowledgePath(path: string): { scope: string; scopeId: string | null; pageKey: string } | null { if (!path.endsWith('.md')) { return null; } const segments = path.split('/'); - if (segments[0] !== 'knowledge') { + if (segments[0] !== 'wiki') { return null; } const rest = segments.slice(1); diff --git a/packages/context/src/wiki/local-knowledge.test.ts b/packages/context/src/wiki/local-knowledge.test.ts index 54bd3771..09d61a3c 100644 --- a/packages/context/src/wiki/local-knowledge.test.ts +++ b/packages/context/src/wiki/local-knowledge.test.ts @@ -35,7 +35,7 @@ describe('local knowledge helpers', () => { await rm(tempDir, { recursive: true, force: true }); }); - it('writes, reads, lists, and searches global knowledge pages', async () => { + it('writes, reads, lists, and searches global wiki pages', async () => { const write = await writeLocalKnowledgePage(project, { key: 'metrics-revenue', scope: 'GLOBAL', @@ -46,7 +46,7 @@ describe('local knowledge helpers', () => { slRefs: ['orders'], }); - expect(write.path).toBe('knowledge/global/metrics-revenue.md'); + expect(write.path).toBe('wiki/global/metrics-revenue.md'); expect(write.operation).toBe('write'); await expect(readLocalKnowledgePage(project, { key: 'metrics-revenue', userId: 'local' })).resolves.toMatchObject({ @@ -62,7 +62,7 @@ describe('local knowledge helpers', () => { await expect(listLocalKnowledgePages(project, { userId: 'local' })).resolves.toEqual([ { key: 'metrics-revenue', - path: 'knowledge/global/metrics-revenue.md', + path: 'wiki/global/metrics-revenue.md', scope: 'GLOBAL', summary: 'Revenue metric definition', }, @@ -72,7 +72,7 @@ describe('local knowledge helpers', () => { expect(search).toEqual([ expect.objectContaining({ key: 'metrics-revenue', - path: 'knowledge/global/metrics-revenue.md', + path: 'wiki/global/metrics-revenue.md', scope: 'GLOBAL', score: expect.any(Number), matchReasons: expect.arrayContaining(['lexical']), @@ -195,7 +195,7 @@ describe('local knowledge helpers', () => { fingerprints: ['fp_paid_orders'], }); - const raw = await project.fileStore.readFile('knowledge/global/monthly-paid-orders.md'); + const raw = await project.fileStore.readFile('wiki/global/monthly-paid-orders.md'); expect(raw.content).toContain('source: historic-sql'); expect(raw.content).toContain('intent: Monthly paid order count'); expect(raw.content).toContain(['tables:', ' - analytics.orders'].join('\n')); @@ -245,4 +245,29 @@ describe('local knowledge helpers', () => { ).rejects.toThrow('Invalid wiki key "orbit/company-overview". Wiki keys must be flat; use "orbit-company-overview".'); }); + it('ignores nested historic-SQL legacy paths when listing local wiki pages', async () => { + await writeLocalKnowledgePage(project, { + key: 'historic-sql-paid-orders', + scope: 'GLOBAL', + summary: 'Flat historic SQL page', + content: 'Flat page body.', + tags: ['historic-sql'], + }); + await project.fileStore.writeFile( + 'wiki/global/historic-sql/paid-orders.md', + '---\nsummary: Nested historic SQL page\nusage_mode: auto\n---\n\nNested body\n', + 'Test', + 'test@example.com', + 'Write nested legacy page', + ); + + await expect(listLocalKnowledgePages(project, { userId: 'local' })).resolves.toEqual([ + { + key: 'historic-sql-paid-orders', + path: 'wiki/global/historic-sql-paid-orders.md', + scope: 'GLOBAL', + summary: 'Flat historic SQL page', + }, + ]); + }); }); diff --git a/packages/context/src/wiki/local-knowledge.ts b/packages/context/src/wiki/local-knowledge.ts index 5d1314a8..f9b25fb1 100644 --- a/packages/context/src/wiki/local-knowledge.ts +++ b/packages/context/src/wiki/local-knowledge.ts @@ -75,13 +75,13 @@ function stringArray(value: unknown): string[] { function knowledgePath(scope: LocalKnowledgeScope, userId: string | undefined, key: string): string { const safeKey = assertFlatWikiKey(key); if (scope === 'GLOBAL') { - return `knowledge/global/${safeKey}.md`; + return `wiki/global/${safeKey}.md`; } - return `knowledge/user/${assertSafePathToken('user id', userId ?? 'local')}/${safeKey}.md`; + return `wiki/user/${assertSafePathToken('user id', userId ?? 'local')}/${safeKey}.md`; } function keyFromKnowledgePath(path: string, scope: LocalKnowledgeScope, userId: string): string | null { - const prefix = scope === 'GLOBAL' ? 'knowledge/global/' : `knowledge/user/${assertSafePathToken('user id', userId)}/`; + const prefix = scope === 'GLOBAL' ? 'wiki/global/' : `wiki/user/${assertSafePathToken('user id', userId)}/`; const key = path.slice(prefix.length).replace(/\.md$/, ''); if (isFlatWikiKey(key)) { return key; @@ -158,7 +158,7 @@ export async function writeLocalKnowledgePage( serializeKnowledgePage(input), LOCAL_AUTHOR, LOCAL_AUTHOR_EMAIL, - `Write knowledge page: ${input.key}`, + `Write wiki page: ${input.key}`, ); } @@ -181,7 +181,7 @@ export async function listLocalKnowledgePages( const userId = input.userId ?? 'local'; const pages: LocalKnowledgeSummary[] = []; for (const scope of ['GLOBAL', 'USER'] as const) { - const root = scope === 'GLOBAL' ? 'knowledge/global' : `knowledge/user/${assertSafePathToken('user id', userId)}`; + const root = scope === 'GLOBAL' ? 'wiki/global' : `wiki/user/${assertSafePathToken('user id', userId)}`; const listed = await project.fileStore.listFiles(root); for (const path of listed.files.filter((file) => file.endsWith('.md')).sort()) { const key = keyFromKnowledgePath(path, scope, userId); diff --git a/packages/context/src/wiki/sqlite-knowledge-index.test.ts b/packages/context/src/wiki/sqlite-knowledge-index.test.ts index 620702a1..2a45573d 100644 --- a/packages/context/src/wiki/sqlite-knowledge-index.test.ts +++ b/packages/context/src/wiki/sqlite-knowledge-index.test.ts @@ -19,7 +19,7 @@ describe('SqliteKnowledgeIndex', () => { function page(overrides: Partial = {}): SqliteKnowledgeIndexPage { return { - path: 'knowledge/global/revenue.md', + path: 'wiki/global/revenue.md', key: 'revenue', scope: 'GLOBAL', summary: 'Revenue definition', @@ -36,7 +36,7 @@ describe('SqliteKnowledgeIndex', () => { index.sync([ page(), page({ - path: 'knowledge/global/support.md', + path: 'wiki/global/support.md', key: 'support', summary: 'Support queue', content: 'Tickets are grouped by priority.', @@ -47,8 +47,8 @@ describe('SqliteKnowledgeIndex', () => { await expect(access(dbPath)).resolves.toBeUndefined(); expect(index.searchLexicalCandidates({ queryText: 'paid order', limit: 10 })).toEqual([ expect.objectContaining({ - id: 'knowledge/global/revenue.md', - path: 'knowledge/global/revenue.md', + id: 'wiki/global/revenue.md', + path: 'wiki/global/revenue.md', rank: 1, rawScore: expect.any(Number), }), @@ -57,7 +57,7 @@ describe('SqliteKnowledgeIndex', () => { it('removes stale rows when the Markdown source list changes', () => { const index = new SqliteKnowledgeIndex({ dbPath }); - index.rebuild([page(), page({ path: 'knowledge/global/churn.md', key: 'churn', content: 'Churn risk.' })]); + index.rebuild([page(), page({ path: 'wiki/global/churn.md', key: 'churn', content: 'Churn risk.' })]); expect(index.search('churn', 10)).toHaveLength(1); index.rebuild([page()]); @@ -67,12 +67,12 @@ describe('SqliteKnowledgeIndex', () => { it('exposes existing search text and embedding state for incremental refresh', () => { const index = new SqliteKnowledgeIndex({ dbPath }); - index.sync([page({ path: 'knowledge/global/revenue.md', key: 'revenue', embedding: [1, 0] })]); + index.sync([page({ path: 'wiki/global/revenue.md', key: 'revenue', embedding: [1, 0] })]); expect(index.getExistingPages()).toEqual( new Map([ [ - 'knowledge/global/revenue.md', + 'wiki/global/revenue.md', expect.objectContaining({ searchText: expect.stringContaining('Revenue definition'), embedding: [1, 0], @@ -84,29 +84,29 @@ describe('SqliteKnowledgeIndex', () => { it('does not treat empty embeddings as indexed semantic vectors', () => { const index = new SqliteKnowledgeIndex({ dbPath }); - index.sync([page({ path: 'knowledge/global/revenue.md', key: 'revenue', embedding: [] })]); + index.sync([page({ path: 'wiki/global/revenue.md', key: 'revenue', embedding: [] })]); - expect(index.getExistingPages().get('knowledge/global/revenue.md')?.embedding).toBeNull(); + expect(index.getExistingPages().get('wiki/global/revenue.md')?.embedding).toBeNull(); expect(index.searchSemanticCandidates({ queryEmbedding: [1, 0], limit: 10 })).toEqual([]); }); it('returns semantic lane candidates from stored page embeddings', () => { const index = new SqliteKnowledgeIndex({ dbPath }); index.sync([ - page({ path: 'knowledge/global/revenue.md', key: 'revenue', embedding: [1, 0] }), - page({ path: 'knowledge/global/support.md', key: 'support', summary: 'Support queue', embedding: [0, 1] }), + page({ path: 'wiki/global/revenue.md', key: 'revenue', embedding: [1, 0] }), + page({ path: 'wiki/global/support.md', key: 'support', summary: 'Support queue', embedding: [0, 1] }), ]); expect(index.searchSemanticCandidates({ queryEmbedding: [1, 0], limit: 10 })).toEqual([ expect.objectContaining({ - id: 'knowledge/global/revenue.md', - path: 'knowledge/global/revenue.md', + id: 'wiki/global/revenue.md', + path: 'wiki/global/revenue.md', rank: 1, rawScore: 1, }), expect.objectContaining({ - id: 'knowledge/global/support.md', - path: 'knowledge/global/support.md', + id: 'wiki/global/support.md', + path: 'wiki/global/support.md', rank: 2, rawScore: 0, }), diff --git a/packages/context/src/wiki/tools/wiki-remove.tool.ts b/packages/context/src/wiki/tools/wiki-remove.tool.ts index 7cb56e7d..4d4c1333 100644 --- a/packages/context/src/wiki/tools/wiki-remove.tool.ts +++ b/packages/context/src/wiki/tools/wiki-remove.tool.ts @@ -36,7 +36,7 @@ export class WikiRemoveTool extends BaseTool { } get description(): string { - return `Remove a knowledge page that is no longer relevant.`; + return `Remove a wiki page that is no longer relevant.`; } get inputSchema() { diff --git a/packages/context/src/wiki/tools/wiki-search.tool.test.ts b/packages/context/src/wiki/tools/wiki-search.tool.test.ts index 33bd752b..24840a4f 100644 --- a/packages/context/src/wiki/tools/wiki-search.tool.test.ts +++ b/packages/context/src/wiki/tools/wiki-search.tool.test.ts @@ -7,7 +7,7 @@ describe('WikiSearchTool', () => { results: [ { key: 'metrics-revenue', - path: 'knowledge/global/metrics-revenue.md', + path: 'wiki/global/metrics-revenue.md', scope: 'GLOBAL' as const, summary: 'Revenue metric definition', score: 0.02459016393442623, @@ -28,7 +28,7 @@ describe('WikiSearchTool', () => { results: [ { blockKey: 'metrics-revenue', - path: 'knowledge/global/metrics-revenue.md', + path: 'wiki/global/metrics-revenue.md', summary: 'Revenue metric definition', score: 0.02459016393442623, matchReasons: ['lexical', 'token'], diff --git a/packages/context/src/wiki/tools/wiki-write.tool.ts b/packages/context/src/wiki/tools/wiki-write.tool.ts index 70668950..9cd457a8 100644 --- a/packages/context/src/wiki/tools/wiki-write.tool.ts +++ b/packages/context/src/wiki/tools/wiki-write.tool.ts @@ -147,7 +147,7 @@ export class WikiWriteTool extends BaseTool { get description(): string { return ` -Create or update a knowledge page. Provide content for create/rewrite, or replacements for targeted edits. +Create or update a wiki page. Provide content for create/rewrite, or replacements for targeted edits. For existing pages, you may provide only frontmatter fields such as summary, tags, refs, or sl_refs to update metadata while preserving content. tags/refs/sl_refs use REPLACE semantics: omit to keep existing on update, [] to clear, [values] to set. Keys must be flat file names, not directory paths. Use tags/source frontmatter for grouping. diff --git a/scripts/package-artifacts.mjs b/scripts/package-artifacts.mjs index 7e184dde..b74b4277 100644 --- a/scripts/package-artifacts.mjs +++ b/scripts/package-artifacts.mjs @@ -663,9 +663,9 @@ try { ); await writeSqliteWarehouse(projectDir); - await mkdir(join(projectDir, 'knowledge', 'global'), { recursive: true }); + await mkdir(join(projectDir, 'wiki', 'global'), { recursive: true }); await writeFile( - join(projectDir, 'knowledge', 'global', 'revenue.md'), + join(projectDir, 'wiki', 'global', 'revenue.md'), [ '---', 'summary: Paid order value', @@ -698,12 +698,12 @@ try { assert.equal(wikiSearchJson.kind, 'list'); assert.equal(wikiSearchJson.data.items.length, 1); assert.equal(wikiSearchJson.data.items[0].key, 'revenue'); - assert.equal(wikiSearchJson.data.items[0].path, 'knowledge/global/revenue.md'); + assert.equal(wikiSearchJson.data.items[0].path, 'wiki/global/revenue.md'); assert.equal(typeof wikiSearchJson.data.items[0].score, 'number'); requireIncludes(wikiSearchJson.data.items[0].matchReasons, 'lexical', 'wiki search match reasons'); process.stdout.write('ktx wiki search hybrid metadata verified\\n'); await access(join(projectDir, '.ktx', 'db.sqlite')); - process.stdout.write('SQLite knowledge index: ' + join(projectDir, '.ktx', 'db.sqlite') + '\\n'); + process.stdout.write('SQLite wiki index: ' + join(projectDir, '.ktx', 'db.sqlite') + '\\n'); const slYaml = [ 'name: orders', diff --git a/scripts/package-artifacts.test.mjs b/scripts/package-artifacts.test.mjs index 7694ddc3..06671d7c 100644 --- a/scripts/package-artifacts.test.mjs +++ b/scripts/package-artifacts.test.mjs @@ -456,7 +456,7 @@ describe('verification snippets', () => { assert.doesNotMatch(source, /@modelcontextprotocol/); assert.doesNotMatch(source, /startSemanticDaemon/); assert.match(source, /run\('pnpm', \[\s*'exec',\s*'ktx',\s*'setup'/); - assert.match(source, /knowledge', 'global', 'revenue\.md'/); + assert.match(source, /wiki', 'global', 'revenue\.md'/); assert.match(source, /run\('pnpm', \[\s*'exec',\s*'ktx',\s*'wiki',\s*'search'/); assert.match(source, /semantic-layer', 'warehouse', 'orders\.yaml'/); assert.match(source, /run\('pnpm', \[\s*'exec',\s*'ktx',\s*'sl',\s*'search',\s*'orders'/); @@ -497,7 +497,7 @@ describe('verification snippets', () => { assert.match(source, /mode: deterministic/); assert.match(source, /run\('pnpm', \['exec', 'ktx', 'ingest', 'run'/); assert.match(source, /access\(join\(projectDir, '\.ktx', 'db\.sqlite'\)\)/); - assert.match(source, /SQLite knowledge index/); + assert.match(source, /SQLite wiki index/); assert.match(source, /ktx ingest run requires llm\\.provider\\.backend: anthropic, vertex, or gateway/); assert.match(source, /ktx ingest provider guard verified/); });