mirror of
https://github.com/Kaelio/ktx.git
synced 2026-06-25 08:48:08 +02:00
Initial open-source release
This commit is contained in:
Andrey Avtomonov
commit
1a42152e6f
1199 changed files with 257054 additions and 0 deletions
29
packages/context/src/wiki/index.ts
Normal file
29
packages/context/src/wiki/index.ts
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
export { buildKnowledgeSearchText } from './knowledge-search-text.js';
|
||||
export { KnowledgeWikiService } from './knowledge-wiki.service.js';
|
||||
export * from './local-knowledge.js';
|
||||
export type {
|
||||
KnowledgeEventPort,
|
||||
KnowledgeGitDiffPort,
|
||||
KnowledgeIndexPort,
|
||||
UpsertPageParams,
|
||||
WikiFileStorePort,
|
||||
} from './ports.js';
|
||||
export type {
|
||||
ExistingKnowledgeIndexPage,
|
||||
SqliteKnowledgeIndexOptions,
|
||||
SqliteKnowledgeIndexPage,
|
||||
SqliteKnowledgeIndexSearchResult,
|
||||
WikiSqliteLaneCandidate,
|
||||
} from './sqlite-knowledge-index.js';
|
||||
export { SqliteKnowledgeIndex } from './sqlite-knowledge-index.js';
|
||||
export * from './tools/index.js';
|
||||
export type {
|
||||
HistoricSqlWikiUsageFrontmatter,
|
||||
WikiFrontmatter,
|
||||
WikiPage,
|
||||
WikiPageWithScope,
|
||||
WikiScope,
|
||||
WikiSearchLaneSummary,
|
||||
WikiSearchMatchReason,
|
||||
WikiSearchMetadata,
|
||||
} from './types.js';
|
||||
7
packages/context/src/wiki/knowledge-search-text.ts
Normal file
7
packages/context/src/wiki/knowledge-search-text.ts
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
export function buildKnowledgeSearchText(blockKey: string, summary: string, content: string, tags?: string[]): string {
|
||||
const parts = [blockKey, summary, content];
|
||||
if (tags && tags.length > 0) {
|
||||
parts.push(tags.join(' '));
|
||||
}
|
||||
return parts.join('\n');
|
||||
}
|
||||
118
packages/context/src/wiki/knowledge-wiki.service.test.ts
Normal file
118
packages/context/src/wiki/knowledge-wiki.service.test.ts
Normal file
|
|
@ -0,0 +1,118 @@
|
|||
import { describe, expect, it, vi } from 'vitest';
|
||||
import { KnowledgeWikiService, type WikiFrontmatter } from './knowledge-wiki.service.js';
|
||||
|
||||
function makeService() {
|
||||
const pagesRepository: Record<string, ReturnType<typeof vi.fn>> = {
|
||||
upsertPage: vi.fn().mockResolvedValue(undefined),
|
||||
deleteByKey: vi.fn().mockResolvedValue(undefined),
|
||||
deleteByScope: vi.fn().mockResolvedValue(undefined),
|
||||
deleteStale: vi.fn().mockResolvedValue(undefined),
|
||||
getExistingSearchTexts: vi.fn().mockResolvedValue(new Map()),
|
||||
applyDiffTransactional: vi.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
const embeddingService = {
|
||||
computeEmbedding: vi.fn().mockResolvedValue([0.1, 0.2, 0.3]),
|
||||
computeEmbeddingsBulk: vi.fn().mockResolvedValue([]),
|
||||
maxBatchSize: 16,
|
||||
};
|
||||
const configService = {
|
||||
forWorktree: vi.fn().mockReturnValue({
|
||||
writeFile: vi.fn(),
|
||||
readFile: vi.fn(),
|
||||
deleteFile: vi.fn(),
|
||||
listFiles: vi.fn(),
|
||||
getFileHistory: vi.fn(),
|
||||
}),
|
||||
writeFile: vi.fn(),
|
||||
readFile: vi.fn(),
|
||||
deleteFile: vi.fn(),
|
||||
listFiles: vi.fn(),
|
||||
getFileHistory: vi.fn(),
|
||||
};
|
||||
const gitService = {
|
||||
diffNameStatus: vi.fn().mockResolvedValue([]),
|
||||
getFileAtCommit: vi.fn().mockResolvedValue(''),
|
||||
};
|
||||
const service = new KnowledgeWikiService(
|
||||
configService as any,
|
||||
embeddingService as any,
|
||||
pagesRepository as any,
|
||||
gitService as any,
|
||||
);
|
||||
return { service, pagesRepository, embeddingService, configService, gitService };
|
||||
}
|
||||
|
||||
const fm: WikiFrontmatter = { summary: 'sum', usage_mode: 'auto' };
|
||||
|
||||
describe('KnowledgeWikiService.forWorktree isolation', () => {
|
||||
it('syncSinglePage in worktree scope does not call pagesRepository.upsertPage', async () => {
|
||||
const { service, pagesRepository, embeddingService } = makeService();
|
||||
const scoped = service.forWorktree('/tmp/fake-worktree');
|
||||
|
||||
await scoped.syncSinglePage('GLOBAL', null, 'key', fm, 'body');
|
||||
|
||||
expect(pagesRepository.upsertPage).not.toHaveBeenCalled();
|
||||
expect(embeddingService.computeEmbedding).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('deleteFromIndex in worktree scope does not call pagesRepository.deleteByKey', async () => {
|
||||
const { service, pagesRepository } = makeService();
|
||||
const scoped = service.forWorktree('/tmp/fake-worktree');
|
||||
|
||||
await scoped.deleteFromIndex('GLOBAL', null, 'key');
|
||||
|
||||
expect(pagesRepository.deleteByKey).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('syncSinglePage in main scope still calls pagesRepository.upsertPage', async () => {
|
||||
const { service, pagesRepository } = makeService();
|
||||
|
||||
await service.syncSinglePage('GLOBAL', null, 'key', fm, 'body');
|
||||
|
||||
expect(pagesRepository.upsertPage).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('KnowledgeWikiService.syncFromCommit', () => {
|
||||
it('applies upserts for added/modified files and deletes for removed files in a single transactional batch', async () => {
|
||||
const { service, pagesRepository, gitService } = makeService();
|
||||
|
||||
gitService.diffNameStatus.mockResolvedValue([
|
||||
{ status: 'A', path: 'knowledge/global/new-page.md' },
|
||||
{ status: 'M', path: 'knowledge/global/changed-page.md' },
|
||||
{ status: 'D', path: 'knowledge/global/gone-page.md' },
|
||||
]);
|
||||
gitService.getFileAtCommit.mockImplementation((path: string) => {
|
||||
if (path.endsWith('new-page.md')) {
|
||||
return Promise.resolve('---\nsummary: new\nusage_mode: auto\n---\n\nbody-new\n');
|
||||
}
|
||||
if (path.endsWith('changed-page.md')) {
|
||||
return Promise.resolve('---\nsummary: changed\nusage_mode: auto\n---\n\nbody-changed\n');
|
||||
}
|
||||
return Promise.reject(new Error(`unexpected getFileAtCommit path: ${path}`));
|
||||
});
|
||||
|
||||
await service.syncFromCommit('sha-before', 'sha-after', 'run-uuid');
|
||||
|
||||
expect(pagesRepository.applyDiffTransactional).toHaveBeenCalledTimes(1);
|
||||
const call = pagesRepository.applyDiffTransactional.mock.calls[0][0];
|
||||
expect(call.runId).toBe('run-uuid');
|
||||
expect(call.upserts).toHaveLength(2);
|
||||
expect(call.upserts).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ scope: 'GLOBAL', pageKey: 'new-page', summary: 'new' }),
|
||||
expect.objectContaining({ scope: 'GLOBAL', pageKey: 'changed-page', summary: 'changed' }),
|
||||
]),
|
||||
);
|
||||
expect(call.deletes).toEqual([{ scope: 'GLOBAL', scopeId: null, pageKey: 'gone-page' }]);
|
||||
});
|
||||
|
||||
it('is a no-op when the diff between shas has no knowledge changes', async () => {
|
||||
const { service, pagesRepository, gitService } = makeService();
|
||||
gitService.diffNameStatus.mockResolvedValue([]);
|
||||
|
||||
await service.syncFromCommit('sha-before', 'sha-after', 'run-uuid');
|
||||
|
||||
expect(pagesRepository.applyDiffTransactional).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
437
packages/context/src/wiki/knowledge-wiki.service.ts
Normal file
437
packages/context/src/wiki/knowledge-wiki.service.ts
Normal file
|
|
@ -0,0 +1,437 @@
|
|||
import { createHash } from 'node:crypto';
|
||||
import YAML from 'yaml';
|
||||
import type { KloEmbeddingPort, KloFileStorePort, KloLogger } from '../core/index.js';
|
||||
import { noopLogger } from '../core/index.js';
|
||||
import { buildKnowledgeSearchText } from './knowledge-search-text.js';
|
||||
import type { KnowledgeGitDiffPort, KnowledgeIndexPort, UpsertPageParams } from './ports.js';
|
||||
import type { WikiFrontmatter, WikiPage, WikiPageWithScope } from './types.js';
|
||||
|
||||
const WIKI_PREFIX = 'knowledge';
|
||||
|
||||
export type { WikiFrontmatter };
|
||||
|
||||
export class KnowledgeWikiService {
|
||||
private isWorktreeScoped = false;
|
||||
|
||||
constructor(
|
||||
private readonly configService: KloFileStorePort,
|
||||
private readonly embeddingService: KloEmbeddingPort,
|
||||
private readonly pagesRepository: KnowledgeIndexPort,
|
||||
private readonly gitService: KnowledgeGitDiffPort,
|
||||
private readonly logger: KloLogger = noopLogger,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Return a clone of this service whose disk writes go through a worktree-scoped
|
||||
* ConfigService AND whose DB-index writes are no-ops. Used by memory-agent
|
||||
* session worktrees so wiki tool calls during the LLM loop land on the session
|
||||
* branch. The shared `knowledge` table is only touched once per run, atomically,
|
||||
* via `syncFromCommit` after Stage 6 squashes the branch into main.
|
||||
*/
|
||||
forWorktree(workdir: string): KnowledgeWikiService {
|
||||
return new KnowledgeWikiService(
|
||||
this.configService.forWorktree(workdir) as KloFileStorePort,
|
||||
this.embeddingService,
|
||||
this.pagesRepository,
|
||||
this.gitService,
|
||||
this.logger,
|
||||
).markWorktreeScoped();
|
||||
}
|
||||
|
||||
private markWorktreeScoped(): KnowledgeWikiService {
|
||||
this.isWorktreeScoped = true;
|
||||
return this;
|
||||
}
|
||||
|
||||
// ── File paths ────────────────────────────────────────────────
|
||||
|
||||
private scopeDir(scope: string, scopeId?: string | null): string {
|
||||
if (scope === 'GLOBAL') {
|
||||
return `${WIKI_PREFIX}/global`;
|
||||
}
|
||||
return `${WIKI_PREFIX}/user/${scopeId}`;
|
||||
}
|
||||
|
||||
pagePath(scope: string, scopeId: string | null | undefined, pageKey: string): string {
|
||||
return `${this.scopeDir(scope, scopeId)}/${pageKey}.md`;
|
||||
}
|
||||
|
||||
// ── Parsing / serialization ───────────────────────────────────
|
||||
|
||||
parsePage(raw: string): { frontmatter: WikiFrontmatter; content: string } {
|
||||
const match = raw.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
|
||||
if (!match) {
|
||||
throw new Error('Invalid wiki page: missing YAML frontmatter');
|
||||
}
|
||||
const frontmatter = YAML.parse(match[1]) as WikiFrontmatter;
|
||||
const content = match[2].trim();
|
||||
return { frontmatter, content };
|
||||
}
|
||||
|
||||
serializePage(frontmatter: WikiFrontmatter, content: string): string {
|
||||
const yaml = YAML.stringify(frontmatter, { indent: 2, lineWidth: 0 }).trimEnd();
|
||||
return `---\n${yaml}\n---\n\n${content}\n`;
|
||||
}
|
||||
|
||||
// ── File CRUD ─────────────────────────────────────────────────
|
||||
|
||||
async writePage(
|
||||
scope: string,
|
||||
scopeId: string | null | undefined,
|
||||
pageKey: string,
|
||||
frontmatter: WikiFrontmatter,
|
||||
content: string,
|
||||
author: string,
|
||||
authorEmail: string,
|
||||
commitMessage?: string,
|
||||
options?: { skipLock?: boolean },
|
||||
) {
|
||||
const path = this.pagePath(scope, scopeId, pageKey);
|
||||
const serialized = this.serializePage(frontmatter, content);
|
||||
const message = commitMessage ?? `Update knowledge page: ${pageKey}`;
|
||||
return this.configService.writeFile(path, serialized, author, authorEmail, message, {
|
||||
skipLock: options?.skipLock,
|
||||
});
|
||||
}
|
||||
|
||||
async readPage(scope: string, scopeId: string | null | undefined, pageKey: string): Promise<WikiPage | null> {
|
||||
const path = this.pagePath(scope, scopeId, pageKey);
|
||||
try {
|
||||
const result = await this.configService.readFile(path);
|
||||
const { frontmatter, content } = this.parsePage(result.content);
|
||||
return { pageKey, frontmatter, content };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async deletePage(
|
||||
scope: string,
|
||||
scopeId: string | null | undefined,
|
||||
pageKey: string,
|
||||
author: string,
|
||||
authorEmail: string,
|
||||
) {
|
||||
const path = this.pagePath(scope, scopeId, pageKey);
|
||||
try {
|
||||
return await this.configService.deleteFile(path, author, authorEmail, `Remove knowledge page: ${pageKey}`);
|
||||
} catch (error) {
|
||||
// Check if the file actually exists — if not, deletion is a no-op
|
||||
try {
|
||||
await this.configService.readFile(path);
|
||||
} catch {
|
||||
// File doesn't exist, nothing to delete
|
||||
return null;
|
||||
}
|
||||
// File exists but delete failed — propagate so callers don't assume success
|
||||
this.logger.error(`Failed to delete wiki page at ${path} despite file existing`);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async listPageKeys(scope: string, scopeId?: string | null): Promise<string[]> {
|
||||
const dir = this.scopeDir(scope, scopeId);
|
||||
try {
|
||||
const result = await this.configService.listFiles(dir);
|
||||
return result.files
|
||||
.filter((f) => f.endsWith('.md'))
|
||||
.map((f) => {
|
||||
// Strip the directory prefix and .md extension
|
||||
const name = f.replace(`${dir}/`, '').replace(/\.md$/, '');
|
||||
return name;
|
||||
})
|
||||
.filter((name) => !name.includes('/'));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
async getPageHistory(scope: string, scopeId: string | null | undefined, pageKey: string) {
|
||||
const path = this.pagePath(scope, scopeId, pageKey);
|
||||
return this.configService.getFileHistory(path);
|
||||
}
|
||||
|
||||
// ── Read page for user (USER scope first, fallback to GLOBAL) ─
|
||||
|
||||
async readPageForUser(userId: string, pageKey: string): Promise<WikiPageWithScope | null> {
|
||||
// Try USER scope first
|
||||
const userPage = await this.readPage('USER', userId, pageKey);
|
||||
if (userPage) {
|
||||
return { ...userPage, scope: 'USER' };
|
||||
}
|
||||
// Fall back to GLOBAL
|
||||
const globalPage = await this.readPage('GLOBAL', null, pageKey);
|
||||
if (globalPage) {
|
||||
return { ...globalPage, scope: 'GLOBAL' };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Write a page verbatim from raw .md text (front-matter + body) after parse-validation.
|
||||
* Preserves the user's exact formatting (raw mode source-of-truth).
|
||||
*/
|
||||
async writeRawPageAndSync(
|
||||
scope: string,
|
||||
scopeId: string | null | undefined,
|
||||
pageKey: string,
|
||||
rawContent: string,
|
||||
author: string,
|
||||
authorEmail: string,
|
||||
commitMessage?: string,
|
||||
): Promise<{ frontmatter: WikiFrontmatter; content: string }> {
|
||||
const parsed = this.parsePage(rawContent);
|
||||
if (!parsed.frontmatter.summary || String(parsed.frontmatter.summary).trim().length === 0) {
|
||||
throw new Error('Front-matter field "summary" is required');
|
||||
}
|
||||
const validModes = ['always', 'auto', 'never'];
|
||||
if (!validModes.includes(parsed.frontmatter.usage_mode)) {
|
||||
throw new Error(`Front-matter field "usage_mode" must be one of: ${validModes.join(', ')}`);
|
||||
}
|
||||
|
||||
const path = this.pagePath(scope, scopeId, pageKey);
|
||||
await this.configService.writeFile(
|
||||
path,
|
||||
rawContent,
|
||||
author,
|
||||
authorEmail,
|
||||
commitMessage ?? `Update knowledge page (raw): ${pageKey}`,
|
||||
);
|
||||
await this.syncSinglePage(scope, scopeId, pageKey, parsed.frontmatter, parsed.content);
|
||||
return parsed;
|
||||
}
|
||||
|
||||
/**
|
||||
* Write a wiki page and then sync it to the DB search index.
|
||||
* Chains the two operations so the index is only updated after the file write succeeds.
|
||||
*/
|
||||
async writePageAndSync(
|
||||
scope: string,
|
||||
scopeId: string | null | undefined,
|
||||
pageKey: string,
|
||||
frontmatter: WikiFrontmatter,
|
||||
content: string,
|
||||
author: string,
|
||||
authorEmail: string,
|
||||
commitMessage?: string,
|
||||
): Promise<void> {
|
||||
await this.writePage(scope, scopeId, pageKey, frontmatter, content, author, authorEmail, commitMessage);
|
||||
const serialized = this.serializePage(frontmatter, content);
|
||||
const contentHash = createHash('sha256').update(serialized).digest('hex');
|
||||
await this.syncSinglePage(scope, scopeId, pageKey, frontmatter, content, contentHash);
|
||||
}
|
||||
|
||||
// ── Index sync (files → DB) ───────────────────────────────────
|
||||
|
||||
/**
|
||||
* Sync a single page to the DB search index after a write.
|
||||
* Computes search_text and embedding, then upserts to knowledge index.
|
||||
*/
|
||||
async syncSinglePage(
|
||||
scope: string,
|
||||
scopeId: string | null | undefined,
|
||||
pageKey: string,
|
||||
frontmatter: WikiFrontmatter,
|
||||
content: string,
|
||||
contentHash?: string | null,
|
||||
): Promise<void> {
|
||||
if (this.isWorktreeScoped) {
|
||||
// Worktree-scoped writes stay on the session branch only. The shared
|
||||
// knowledge index is updated atomically from the squashed commit diff
|
||||
// after Stage 6 via syncFromCommit().
|
||||
return;
|
||||
}
|
||||
|
||||
const searchText = buildKnowledgeSearchText(pageKey, frontmatter.summary, content, frontmatter.tags);
|
||||
|
||||
let embedding: number[] | null = null;
|
||||
try {
|
||||
embedding = await this.embeddingService.computeEmbedding(searchText);
|
||||
} catch (err) {
|
||||
this.logger.warn(`Embedding failed for page "${pageKey}": ${err instanceof Error ? err.message : String(err)}`);
|
||||
}
|
||||
|
||||
await this.pagesRepository.upsertPage({
|
||||
scope,
|
||||
scopeId: scopeId ?? null,
|
||||
pageKey,
|
||||
summary: frontmatter.summary,
|
||||
usageMode: frontmatter.usage_mode,
|
||||
sortOrder: frontmatter.sort_order ?? 0,
|
||||
searchText,
|
||||
embedding,
|
||||
contentHash: contentHash ?? null,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Full sync: load all pages from disk for a scope, reindex changed pages, clean stale entries.
|
||||
* Mirrors SlSearchService.indexSources() pattern.
|
||||
*/
|
||||
async syncIndex(scope: string, scopeId?: string | null): Promise<void> {
|
||||
const pageKeys = await this.listPageKeys(scope, scopeId);
|
||||
if (pageKeys.length === 0) {
|
||||
await this.pagesRepository.deleteByScope(scope, scopeId ?? null);
|
||||
return;
|
||||
}
|
||||
|
||||
// Load and parse all pages
|
||||
const pages: Array<{ pageKey: string; frontmatter: WikiFrontmatter; content: string; searchText: string }> = [];
|
||||
for (const key of pageKeys) {
|
||||
const page = await this.readPage(scope, scopeId, key);
|
||||
if (page) {
|
||||
const searchText = buildKnowledgeSearchText(key, page.frontmatter.summary, page.content, page.frontmatter.tags);
|
||||
pages.push({ pageKey: key, frontmatter: page.frontmatter, content: page.content, searchText });
|
||||
}
|
||||
}
|
||||
|
||||
// Detect changes
|
||||
const existing = await this.pagesRepository.getExistingSearchTexts(scope, scopeId ?? null);
|
||||
const changedPages = pages.filter((p) => {
|
||||
const ex = existing.get(p.pageKey);
|
||||
return !ex || ex.searchText !== p.searchText || !ex.hasEmbedding;
|
||||
});
|
||||
|
||||
if (changedPages.length === 0) {
|
||||
// Still clean up stale
|
||||
await this.pagesRepository.deleteStale(scope, scopeId ?? null, pageKeys);
|
||||
this.logger.log(`Wiki sync ${scope}: all ${pages.length} pages up to date`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Compute embeddings for changed pages (batched)
|
||||
const changedTexts = changedPages.map((p) => p.searchText);
|
||||
let embeddings: (number[] | null)[];
|
||||
try {
|
||||
const batchSize = this.embeddingService.maxBatchSize;
|
||||
const all: number[][] = [];
|
||||
for (let i = 0; i < changedTexts.length; i += batchSize) {
|
||||
const batch = changedTexts.slice(i, i + batchSize);
|
||||
const batchEmb = await this.embeddingService.computeEmbeddingsBulk(batch);
|
||||
all.push(...batchEmb);
|
||||
}
|
||||
embeddings = all;
|
||||
} catch (err) {
|
||||
this.logger.warn(`Embedding batch failed during sync: ${err instanceof Error ? err.message : String(err)}`);
|
||||
embeddings = changedPages.map(() => null);
|
||||
}
|
||||
|
||||
// Upsert changed pages
|
||||
for (let i = 0; i < changedPages.length; i++) {
|
||||
const p = changedPages[i];
|
||||
await this.pagesRepository.upsertPage({
|
||||
scope,
|
||||
scopeId: scopeId ?? null,
|
||||
pageKey: p.pageKey,
|
||||
summary: p.frontmatter.summary,
|
||||
usageMode: p.frontmatter.usage_mode,
|
||||
sortOrder: p.frontmatter.sort_order ?? 0,
|
||||
searchText: p.searchText,
|
||||
embedding: embeddings[i],
|
||||
});
|
||||
}
|
||||
|
||||
// Clean stale entries
|
||||
await this.pagesRepository.deleteStale(scope, scopeId ?? null, pageKeys);
|
||||
|
||||
this.logger.log(
|
||||
`Wiki sync ${scope}: ${changedPages.length}/${pages.length} reindexed, ${pages.length - changedPages.length} unchanged`,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete a page from the DB index (after file deletion).
|
||||
*/
|
||||
async deleteFromIndex(scope: string, scopeId: string | null | undefined, pageKey: string): Promise<void> {
|
||||
if (this.isWorktreeScoped) {
|
||||
return;
|
||||
}
|
||||
await this.pagesRepository.deleteByKey(scope, scopeId ?? null, pageKey);
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply the diff between two commits on the config repo to the shared
|
||||
* `knowledge` index in a single transaction. Called by the ingest runner
|
||||
* after Stage 6 squashes the session branch into main: the pre-squash main
|
||||
* SHA and the post-squash SHA bracket exactly the set of knowledge-file
|
||||
* changes this run produced.
|
||||
*
|
||||
* Any added/modified file becomes an upsert (tagged with `source_run_id`),
|
||||
* any deleted file becomes a delete. Parsing errors fail the whole
|
||||
* transaction so the shared table stays consistent.
|
||||
*/
|
||||
async syncFromCommit(fromSha: string, toSha: string, runId: string): Promise<void> {
|
||||
const diff = await this.gitService.diffNameStatus(fromSha, toSha, 'knowledge/');
|
||||
if (diff.length === 0) {
|
||||
return;
|
||||
}
|
||||
const upserts: UpsertPageParams[] = [];
|
||||
const deletes: Array<{ scope: string; scopeId: string | null; pageKey: string }> = [];
|
||||
|
||||
for (const entry of diff) {
|
||||
const parsedPath = parseKnowledgePath(entry.path);
|
||||
if (!parsedPath) {
|
||||
this.logger.warn(`[knowledge.sync] skipping unparseable path: ${entry.path}`);
|
||||
continue;
|
||||
}
|
||||
if (entry.status === 'D') {
|
||||
deletes.push(parsedPath);
|
||||
continue;
|
||||
}
|
||||
const content = await this.gitService.getFileAtCommit(entry.path, toSha);
|
||||
const parsed = this.parsePage(content);
|
||||
const searchText = buildKnowledgeSearchText(
|
||||
parsedPath.pageKey,
|
||||
parsed.frontmatter.summary,
|
||||
parsed.content,
|
||||
parsed.frontmatter.tags,
|
||||
);
|
||||
let embedding: number[] | null = null;
|
||||
try {
|
||||
embedding = await this.embeddingService.computeEmbedding(searchText);
|
||||
} catch (err) {
|
||||
this.logger.warn(
|
||||
`[knowledge.sync] embedding failed for ${parsedPath.pageKey}: ${err instanceof Error ? err.message : String(err)}`,
|
||||
);
|
||||
}
|
||||
const contentHash = createHash('sha256').update(content).digest('hex');
|
||||
upserts.push({
|
||||
scope: parsedPath.scope,
|
||||
scopeId: parsedPath.scopeId,
|
||||
pageKey: parsedPath.pageKey,
|
||||
summary: parsed.frontmatter.summary,
|
||||
usageMode: parsed.frontmatter.usage_mode,
|
||||
sortOrder: parsed.frontmatter.sort_order ?? 0,
|
||||
searchText,
|
||||
embedding,
|
||||
contentHash,
|
||||
});
|
||||
}
|
||||
|
||||
await this.pagesRepository.applyDiffTransactional({ runId, upserts, deletes });
|
||||
this.logger.log(`[knowledge.sync] run=${runId} applied ${upserts.length} upsert(s), ${deletes.length} delete(s)`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a `knowledge/<scope>/...` file path into its scope and page key.
|
||||
* `knowledge/global/foo.md` → { scope: 'GLOBAL', scopeId: null, pageKey: 'foo' }
|
||||
* `knowledge/user/<id>/bar.md` → { scope: 'USER', scopeId: '<id>', pageKey: 'bar' }
|
||||
*/
|
||||
function parseKnowledgePath(path: string): { scope: string; scopeId: string | null; pageKey: string } | null {
|
||||
if (!path.endsWith('.md')) {
|
||||
return null;
|
||||
}
|
||||
const segments = path.split('/');
|
||||
if (segments[0] !== 'knowledge') {
|
||||
return null;
|
||||
}
|
||||
const rest = segments.slice(1);
|
||||
if (rest.length === 2 && rest[0] === 'global') {
|
||||
return { scope: 'GLOBAL', scopeId: null, pageKey: rest[1].replace(/\.md$/, '') };
|
||||
}
|
||||
if (rest.length === 3 && rest[0] === 'user') {
|
||||
return { scope: 'USER', scopeId: rest[1], pageKey: rest[2].replace(/\.md$/, '') };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
236
packages/context/src/wiki/local-knowledge.test.ts
Normal file
236
packages/context/src/wiki/local-knowledge.test.ts
Normal file
|
|
@ -0,0 +1,236 @@
|
|||
import { access, mkdtemp, rm } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
import { initKloProject, type KloLocalProject } from '../project/index.js';
|
||||
import {
|
||||
listLocalKnowledgePages,
|
||||
readLocalKnowledgePage,
|
||||
searchLocalKnowledgePages,
|
||||
writeLocalKnowledgePage,
|
||||
} from './local-knowledge.js';
|
||||
|
||||
class FakeEmbeddingPort {
|
||||
readonly maxBatchSize = 16;
|
||||
|
||||
async computeEmbedding(text: string): Promise<number[]> {
|
||||
return text.toLowerCase().includes('semantic revenue') ? [1, 0] : [0, 1];
|
||||
}
|
||||
|
||||
async computeEmbeddingsBulk(texts: string[]): Promise<number[][]> {
|
||||
return Promise.all(texts.map((text) => this.computeEmbedding(text)));
|
||||
}
|
||||
}
|
||||
|
||||
describe('local knowledge helpers', () => {
|
||||
let tempDir: string;
|
||||
let project: KloLocalProject;
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'klo-local-knowledge-'));
|
||||
project = await initKloProject({ projectDir: join(tempDir, 'project'), projectName: 'warehouse' });
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('writes, reads, lists, and searches global knowledge pages', async () => {
|
||||
const write = await writeLocalKnowledgePage(project, {
|
||||
key: 'metrics/revenue',
|
||||
scope: 'GLOBAL',
|
||||
summary: 'Revenue metric definition',
|
||||
content: 'Revenue is recognized when an order is paid.',
|
||||
tags: ['finance'],
|
||||
refs: ['semantic-layer/warehouse/orders.yaml'],
|
||||
slRefs: ['orders'],
|
||||
});
|
||||
|
||||
expect(write.path).toBe('knowledge/global/metrics/revenue.md');
|
||||
expect(write.operation).toBe('write');
|
||||
|
||||
await expect(readLocalKnowledgePage(project, { key: 'metrics/revenue', userId: 'local' })).resolves.toMatchObject({
|
||||
key: 'metrics/revenue',
|
||||
scope: 'GLOBAL',
|
||||
summary: 'Revenue metric definition',
|
||||
content: 'Revenue is recognized when an order is paid.',
|
||||
tags: ['finance'],
|
||||
refs: ['semantic-layer/warehouse/orders.yaml'],
|
||||
slRefs: ['orders'],
|
||||
});
|
||||
|
||||
await expect(listLocalKnowledgePages(project, { userId: 'local' })).resolves.toEqual([
|
||||
{
|
||||
key: 'metrics/revenue',
|
||||
path: 'knowledge/global/metrics/revenue.md',
|
||||
scope: 'GLOBAL',
|
||||
summary: 'Revenue metric definition',
|
||||
},
|
||||
]);
|
||||
|
||||
const search = await searchLocalKnowledgePages(project, { query: 'paid order', userId: 'local' });
|
||||
expect(search).toEqual([
|
||||
expect.objectContaining({
|
||||
key: 'metrics/revenue',
|
||||
path: 'knowledge/global/metrics/revenue.md',
|
||||
scope: 'GLOBAL',
|
||||
score: expect.any(Number),
|
||||
matchReasons: expect.arrayContaining(['lexical']),
|
||||
lanes: expect.arrayContaining([expect.objectContaining({ lane: 'lexical', status: 'available' })]),
|
||||
}),
|
||||
]);
|
||||
expect(search[0]?.score).toBeGreaterThan(0);
|
||||
await expect(access(join(project.projectDir, '.klo', 'db.sqlite'))).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it('adds the token lane alongside lexical wiki matches', async () => {
|
||||
await writeLocalKnowledgePage(project, {
|
||||
key: 'metrics/revenue',
|
||||
scope: 'GLOBAL',
|
||||
summary: 'Revenue metric definition',
|
||||
content: 'Revenue is recognized when an order is paid.',
|
||||
tags: ['finance'],
|
||||
});
|
||||
|
||||
const search = await searchLocalKnowledgePages(project, { query: 'paid---', userId: 'local', limit: 5 });
|
||||
|
||||
expect(search[0]).toMatchObject({
|
||||
key: 'metrics/revenue',
|
||||
matchReasons: expect.arrayContaining(['token']),
|
||||
lanes: expect.arrayContaining([expect.objectContaining({ lane: 'token', status: 'available' })]),
|
||||
});
|
||||
});
|
||||
|
||||
it('uses stored page embeddings when a wiki embedding backend is configured', async () => {
|
||||
await writeLocalKnowledgePage(project, {
|
||||
key: 'metrics/revenue',
|
||||
scope: 'GLOBAL',
|
||||
summary: 'Semantic revenue definition',
|
||||
content: 'Revenue search text.',
|
||||
tags: ['finance'],
|
||||
});
|
||||
await writeLocalKnowledgePage(project, {
|
||||
key: 'support/escalations',
|
||||
scope: 'GLOBAL',
|
||||
summary: 'Support escalation process',
|
||||
content: 'Support search text.',
|
||||
tags: ['operations'],
|
||||
});
|
||||
|
||||
const search = await searchLocalKnowledgePages(project, {
|
||||
query: 'semantic revenue',
|
||||
userId: 'local',
|
||||
limit: 5,
|
||||
embeddingService: new FakeEmbeddingPort(),
|
||||
});
|
||||
|
||||
expect(search[0]).toMatchObject({
|
||||
key: 'metrics/revenue',
|
||||
matchReasons: expect.arrayContaining(['semantic']),
|
||||
lanes: expect.arrayContaining([expect.objectContaining({ lane: 'semantic', status: 'available' })]),
|
||||
});
|
||||
});
|
||||
|
||||
it('reports semantic lane as skipped when wiki embeddings are not configured', async () => {
|
||||
await writeLocalKnowledgePage(project, {
|
||||
key: 'metrics/revenue',
|
||||
scope: 'GLOBAL',
|
||||
summary: 'Revenue metric definition',
|
||||
content: 'Revenue is recognized when an order is paid.',
|
||||
tags: ['finance'],
|
||||
});
|
||||
|
||||
const search = await searchLocalKnowledgePages(project, { query: 'revenue', userId: 'local', limit: 5 });
|
||||
|
||||
expect(search[0]?.lanes).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ lane: 'semantic', status: 'skipped', reason: 'embedding_unconfigured' }),
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it('prefers user knowledge over global pages with the same key', async () => {
|
||||
await writeLocalKnowledgePage(project, {
|
||||
key: 'handoff',
|
||||
scope: 'GLOBAL',
|
||||
summary: 'Global handoff',
|
||||
content: 'Global context.',
|
||||
});
|
||||
await writeLocalKnowledgePage(project, {
|
||||
key: 'handoff',
|
||||
scope: 'USER',
|
||||
userId: 'agent-1',
|
||||
summary: 'User handoff',
|
||||
content: 'User context.',
|
||||
});
|
||||
|
||||
await expect(readLocalKnowledgePage(project, { key: 'handoff', userId: 'agent-1' })).resolves.toMatchObject({
|
||||
scope: 'USER',
|
||||
summary: 'User handoff',
|
||||
});
|
||||
});
|
||||
|
||||
it('serializes historic-SQL frontmatter fields for global pages', async () => {
|
||||
await writeLocalKnowledgePage(project, {
|
||||
key: 'queries/monthly-paid-orders',
|
||||
scope: 'GLOBAL',
|
||||
summary: 'Monthly paid orders',
|
||||
content: '## Monthly paid order count',
|
||||
tags: ['historic-sql', 'query-pattern'],
|
||||
slRefs: ['analytics.orders'],
|
||||
source: 'historic-sql',
|
||||
intent: 'Monthly paid order count',
|
||||
tables: ['analytics.orders'],
|
||||
representativeSql: "SELECT count(*) FROM analytics.orders WHERE status = 'paid'",
|
||||
usage: {
|
||||
executions: 42,
|
||||
distinct_users: 3,
|
||||
first_seen: '2026-02-01',
|
||||
last_seen: '2026-05-04',
|
||||
p50_runtime_ms: 100,
|
||||
p95_runtime_ms: 200,
|
||||
error_rate: 0,
|
||||
rows_produced: 42,
|
||||
},
|
||||
fingerprints: ['fp_paid_orders'],
|
||||
});
|
||||
|
||||
const raw = await project.fileStore.readFile('knowledge/global/queries/monthly-paid-orders.md');
|
||||
expect(raw.content).toContain('source: historic-sql');
|
||||
expect(raw.content).toContain('intent: Monthly paid order count');
|
||||
expect(raw.content).toContain(['tables:', ' - analytics.orders'].join('\n'));
|
||||
expect(raw.content).toContain("representative_sql: SELECT count(*) FROM analytics.orders WHERE status = 'paid'");
|
||||
expect(raw.content).toContain(['usage:', ' executions: 42', ' distinct_users: 3'].join('\n'));
|
||||
expect(raw.content).toContain(['fingerprints:', ' - fp_paid_orders'].join('\n'));
|
||||
});
|
||||
|
||||
it('falls back to Markdown scanning when the config does not select sqlite-fts5', async () => {
|
||||
project.config.storage.search = 'postgres-hybrid';
|
||||
await writeLocalKnowledgePage(project, {
|
||||
key: 'metrics/revenue',
|
||||
scope: 'GLOBAL',
|
||||
summary: 'Revenue metric definition',
|
||||
content: 'Revenue is recognized when an order is paid.',
|
||||
tags: ['finance'],
|
||||
});
|
||||
|
||||
await expect(searchLocalKnowledgePages(project, { query: 'paid order', userId: 'local' })).resolves.toEqual([
|
||||
expect.objectContaining({
|
||||
key: 'metrics/revenue',
|
||||
score: 3,
|
||||
matchReasons: ['token'],
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it('rejects unsafe knowledge keys', async () => {
|
||||
await expect(
|
||||
writeLocalKnowledgePage(project, {
|
||||
key: '../secret',
|
||||
scope: 'GLOBAL',
|
||||
summary: 'bad',
|
||||
content: 'bad',
|
||||
}),
|
||||
).rejects.toThrow('Unsafe knowledge key');
|
||||
});
|
||||
});
|
||||
391
packages/context/src/wiki/local-knowledge.ts
Normal file
391
packages/context/src/wiki/local-knowledge.ts
Normal file
|
|
@ -0,0 +1,391 @@
|
|||
import { join } from 'node:path';
|
||||
import YAML from 'yaml';
|
||||
import type { KloEmbeddingPort, KloFileWriteResult } from '../core/index.js';
|
||||
import type { KloLocalProject } from '../project/index.js';
|
||||
import { HybridSearchCore, type SearchCandidateGenerator } from '../search/index.js';
|
||||
import { buildKnowledgeSearchText } from './knowledge-search-text.js';
|
||||
import { SqliteKnowledgeIndex, type SqliteKnowledgeIndexPage } from './sqlite-knowledge-index.js';
|
||||
import type { HistoricSqlWikiUsageFrontmatter, WikiSearchLaneSummary, WikiSearchMatchReason } from './types.js';
|
||||
|
||||
export type LocalKnowledgeScope = 'GLOBAL' | 'USER';
|
||||
|
||||
export interface LocalKnowledgePage {
|
||||
key: string;
|
||||
path: string;
|
||||
scope: LocalKnowledgeScope;
|
||||
summary: string;
|
||||
content: string;
|
||||
tags: string[];
|
||||
refs: string[];
|
||||
slRefs: string[];
|
||||
}
|
||||
|
||||
export interface LocalKnowledgeSummary {
|
||||
key: string;
|
||||
path: string;
|
||||
scope: LocalKnowledgeScope;
|
||||
summary: string;
|
||||
}
|
||||
|
||||
export interface LocalKnowledgeSearchResult extends LocalKnowledgeSummary {
|
||||
score: number;
|
||||
matchReasons: WikiSearchMatchReason[];
|
||||
lanes?: WikiSearchLaneSummary[];
|
||||
}
|
||||
|
||||
export interface WriteLocalKnowledgePageInput {
|
||||
key: string;
|
||||
scope: LocalKnowledgeScope;
|
||||
userId?: string;
|
||||
summary: string;
|
||||
content: string;
|
||||
tags?: string[];
|
||||
refs?: string[];
|
||||
slRefs?: string[];
|
||||
source?: string;
|
||||
intent?: string;
|
||||
tables?: string[];
|
||||
representativeSql?: string;
|
||||
usage?: HistoricSqlWikiUsageFrontmatter;
|
||||
fingerprints?: string[];
|
||||
}
|
||||
|
||||
const LOCAL_AUTHOR = 'klo';
|
||||
const LOCAL_AUTHOR_EMAIL = 'klo@example.com';
|
||||
|
||||
function assertSafePathToken(kind: string, value: string): string {
|
||||
if (
|
||||
value.trim().length === 0 ||
|
||||
value.includes('..') ||
|
||||
value.includes('\\') ||
|
||||
value.startsWith('/') ||
|
||||
value.startsWith('.') ||
|
||||
value.includes('//')
|
||||
) {
|
||||
throw new Error(`Unsafe ${kind}: ${value}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function assertSafeKnowledgeKey(key: string): string {
|
||||
if (!/^[a-zA-Z0-9][a-zA-Z0-9_/-]*$/.test(key)) {
|
||||
throw new Error(`Unsafe knowledge key: ${key}`);
|
||||
}
|
||||
return assertSafePathToken('knowledge key', key);
|
||||
}
|
||||
|
||||
function stringArray(value: unknown): string[] {
|
||||
return Array.isArray(value) ? value.filter((item): item is string => typeof item === 'string') : [];
|
||||
}
|
||||
|
||||
function knowledgePath(scope: LocalKnowledgeScope, userId: string | undefined, key: string): string {
|
||||
const safeKey = assertSafeKnowledgeKey(key);
|
||||
if (scope === 'GLOBAL') {
|
||||
return `knowledge/global/${safeKey}.md`;
|
||||
}
|
||||
return `knowledge/user/${assertSafePathToken('user id', userId ?? 'local')}/${safeKey}.md`;
|
||||
}
|
||||
|
||||
function keyFromKnowledgePath(path: string, scope: LocalKnowledgeScope, userId: string): string {
|
||||
const prefix = scope === 'GLOBAL' ? 'knowledge/global/' : `knowledge/user/${assertSafePathToken('user id', userId)}/`;
|
||||
return path.slice(prefix.length).replace(/\.md$/, '');
|
||||
}
|
||||
|
||||
function parseKnowledgePage(key: string, path: string, scope: LocalKnowledgeScope, raw: string): LocalKnowledgePage {
|
||||
const match = raw.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
|
||||
if (!match) {
|
||||
return {
|
||||
key,
|
||||
path,
|
||||
scope,
|
||||
summary: '',
|
||||
content: raw.trim(),
|
||||
tags: [],
|
||||
refs: [],
|
||||
slRefs: [],
|
||||
};
|
||||
}
|
||||
|
||||
const frontmatter = (YAML.parse(match[1]) ?? {}) as Record<string, unknown>;
|
||||
return {
|
||||
key,
|
||||
path,
|
||||
scope,
|
||||
summary: typeof frontmatter.summary === 'string' ? frontmatter.summary : '',
|
||||
content: match[2].trim(),
|
||||
tags: stringArray(frontmatter.tags),
|
||||
refs: stringArray(frontmatter.refs),
|
||||
slRefs: stringArray(frontmatter.sl_refs),
|
||||
};
|
||||
}
|
||||
|
||||
function serializeKnowledgePage(input: WriteLocalKnowledgePageInput): string {
|
||||
const frontmatter = {
|
||||
summary: input.summary,
|
||||
tags: input.tags ?? [],
|
||||
refs: input.refs ?? [],
|
||||
sl_refs: input.slRefs ?? [],
|
||||
usage_mode: 'auto',
|
||||
...(input.source === undefined ? {} : { source: input.source }),
|
||||
...(input.intent === undefined ? {} : { intent: input.intent }),
|
||||
...(input.tables === undefined ? {} : { tables: input.tables }),
|
||||
...(input.representativeSql === undefined ? {} : { representative_sql: input.representativeSql }),
|
||||
...(input.usage === undefined ? {} : { usage: input.usage }),
|
||||
...(input.fingerprints === undefined ? {} : { fingerprints: input.fingerprints }),
|
||||
};
|
||||
return `---\n${YAML.stringify(frontmatter, { indent: 2, lineWidth: 0 }).trimEnd()}\n---\n\n${input.content.trim()}\n`;
|
||||
}
|
||||
|
||||
async function readPageAtPath(
|
||||
project: KloLocalProject,
|
||||
key: string,
|
||||
path: string,
|
||||
scope: LocalKnowledgeScope,
|
||||
): Promise<LocalKnowledgePage | null> {
|
||||
try {
|
||||
const result = await project.fileStore.readFile(path);
|
||||
return parseKnowledgePage(key, path, scope, result.content);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function writeLocalKnowledgePage(
|
||||
project: KloLocalProject,
|
||||
input: WriteLocalKnowledgePageInput,
|
||||
): Promise<KloFileWriteResult> {
|
||||
const path = knowledgePath(input.scope, input.userId, input.key);
|
||||
return project.fileStore.writeFile(
|
||||
path,
|
||||
serializeKnowledgePage(input),
|
||||
LOCAL_AUTHOR,
|
||||
LOCAL_AUTHOR_EMAIL,
|
||||
`Write knowledge page: ${input.key}`,
|
||||
);
|
||||
}
|
||||
|
||||
export async function readLocalKnowledgePage(
|
||||
project: KloLocalProject,
|
||||
input: { key: string; userId?: string },
|
||||
): Promise<LocalKnowledgePage | null> {
|
||||
const userPath = knowledgePath('USER', input.userId, input.key);
|
||||
const userPage = await readPageAtPath(project, input.key, userPath, 'USER');
|
||||
if (userPage) {
|
||||
return userPage;
|
||||
}
|
||||
return readPageAtPath(project, input.key, knowledgePath('GLOBAL', undefined, input.key), 'GLOBAL');
|
||||
}
|
||||
|
||||
export async function listLocalKnowledgePages(
|
||||
project: KloLocalProject,
|
||||
input: { userId?: string } = {},
|
||||
): Promise<LocalKnowledgeSummary[]> {
|
||||
const userId = input.userId ?? 'local';
|
||||
const pages: LocalKnowledgeSummary[] = [];
|
||||
for (const scope of ['GLOBAL', 'USER'] as const) {
|
||||
const root = scope === 'GLOBAL' ? 'knowledge/global' : `knowledge/user/${assertSafePathToken('user id', userId)}`;
|
||||
const listed = await project.fileStore.listFiles(root);
|
||||
for (const path of listed.files.filter((file) => file.endsWith('.md')).sort()) {
|
||||
const key = keyFromKnowledgePath(path, scope, userId);
|
||||
const page = await readPageAtPath(project, key, path, scope);
|
||||
if (page) {
|
||||
pages.push({ key, path, scope, summary: page.summary });
|
||||
}
|
||||
}
|
||||
}
|
||||
return pages.sort((left, right) => left.path.localeCompare(right.path));
|
||||
}
|
||||
|
||||
function scorePage(page: LocalKnowledgePage, terms: string[]): number {
|
||||
const haystack = buildKnowledgeSearchText(page.key, page.summary, page.content, page.tags).toLowerCase();
|
||||
return terms.some((term) => haystack.includes(term)) ? 3 : 0;
|
||||
}
|
||||
|
||||
function sqliteKnowledgeDbPath(project: KloLocalProject): string {
|
||||
return join(project.projectDir, '.klo', 'db.sqlite');
|
||||
}
|
||||
|
||||
function pageSearchText(page: LocalKnowledgePage): string {
|
||||
return buildKnowledgeSearchText(page.key, page.summary, page.content, page.tags);
|
||||
}
|
||||
|
||||
async function embeddingForPageSearchText(
|
||||
searchText: string,
|
||||
embeddingService: KloEmbeddingPort | null,
|
||||
): Promise<number[] | null> {
|
||||
if (!embeddingService) {
|
||||
return null;
|
||||
}
|
||||
return embeddingService.computeEmbedding(searchText);
|
||||
}
|
||||
|
||||
function tokenLaneCandidates(pages: LocalKnowledgePage[], terms: string[]) {
|
||||
if (terms.length === 0) {
|
||||
return [];
|
||||
}
|
||||
return pages
|
||||
.map((page) => {
|
||||
const haystack = pageSearchText(page).toLowerCase();
|
||||
const matched = terms.filter((term) => haystack.includes(term)).length;
|
||||
return { page, score: matched / terms.length };
|
||||
})
|
||||
.filter((result) => result.score > 0)
|
||||
.sort((left, right) => right.score - left.score || left.page.path.localeCompare(right.page.path));
|
||||
}
|
||||
|
||||
async function loadAllKnowledgePages(
|
||||
project: KloLocalProject,
|
||||
input: { userId?: string } = {},
|
||||
): Promise<LocalKnowledgePage[]> {
|
||||
const summaries = await listLocalKnowledgePages(project, { userId: input.userId });
|
||||
const pages: LocalKnowledgePage[] = [];
|
||||
for (const summary of summaries) {
|
||||
const page = await readPageAtPath(project, summary.key, summary.path, summary.scope);
|
||||
if (page) {
|
||||
pages.push(page);
|
||||
}
|
||||
}
|
||||
return pages;
|
||||
}
|
||||
|
||||
async function searchLocalKnowledgePagesWithSqlite(
|
||||
project: KloLocalProject,
|
||||
input: { query: string; userId?: string; embeddingService?: KloEmbeddingPort | null; limit?: number },
|
||||
): Promise<LocalKnowledgeSearchResult[]> {
|
||||
const pages = await loadAllKnowledgePages(project, { userId: input.userId });
|
||||
const byPath = new Map(pages.map((page) => [page.path, page]));
|
||||
const embeddingService = input.embeddingService ?? null;
|
||||
const index = new SqliteKnowledgeIndex({ dbPath: sqliteKnowledgeDbPath(project) });
|
||||
const existingPages = index.getExistingPages();
|
||||
const indexPages: SqliteKnowledgeIndexPage[] = [];
|
||||
for (const page of pages) {
|
||||
const searchText = pageSearchText(page);
|
||||
const existing = existingPages.get(page.path);
|
||||
const embedding =
|
||||
existing?.searchText === searchText && existing.embedding
|
||||
? existing.embedding
|
||||
: await embeddingForPageSearchText(searchText, embeddingService).catch(() => null);
|
||||
indexPages.push({
|
||||
path: page.path,
|
||||
key: page.key,
|
||||
scope: page.scope,
|
||||
summary: page.summary,
|
||||
content: page.content,
|
||||
tags: page.tags,
|
||||
embedding,
|
||||
});
|
||||
}
|
||||
|
||||
index.sync(indexPages);
|
||||
|
||||
const finalLimit = input.limit ?? Math.max(1, indexPages.length);
|
||||
const core = new HybridSearchCore();
|
||||
const generators: SearchCandidateGenerator[] = [
|
||||
{
|
||||
lane: 'lexical',
|
||||
async generate(args) {
|
||||
const rows = index.searchLexicalCandidates({
|
||||
queryText: args.queryText,
|
||||
limit: args.laneCandidatePoolLimit,
|
||||
});
|
||||
return {
|
||||
candidates: rows.map((row) => ({ id: row.id, rank: row.rank, rawScore: row.rawScore })),
|
||||
};
|
||||
},
|
||||
},
|
||||
{
|
||||
lane: 'token',
|
||||
async generate(args) {
|
||||
const rows = tokenLaneCandidates(pages, args.normalizedQuery.terms).slice(0, args.laneCandidatePoolLimit);
|
||||
return {
|
||||
candidates: rows.map((row, index) => ({
|
||||
id: row.page.path,
|
||||
rank: index + 1,
|
||||
rawScore: row.score,
|
||||
})),
|
||||
};
|
||||
},
|
||||
},
|
||||
{
|
||||
lane: 'semantic',
|
||||
async generate(args) {
|
||||
if (!embeddingService) {
|
||||
return { status: 'skipped', candidates: [], reason: 'embedding_unconfigured' };
|
||||
}
|
||||
try {
|
||||
const queryEmbedding = await embeddingService.computeEmbedding(args.queryText);
|
||||
const rows = index.searchSemanticCandidates({
|
||||
queryEmbedding,
|
||||
limit: args.laneCandidatePoolLimit,
|
||||
});
|
||||
return {
|
||||
candidates: rows.map((row) => ({ id: row.id, rank: row.rank, rawScore: row.rawScore })),
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
status: 'skipped',
|
||||
candidates: [],
|
||||
reason: `embedding_unhealthy:${error instanceof Error ? error.message : String(error)}`,
|
||||
};
|
||||
}
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const result = await core.search({ queryText: input.query, limit: finalLimit, generators });
|
||||
return result.results
|
||||
.map((fused): LocalKnowledgeSearchResult | null => {
|
||||
const page = byPath.get(fused.id);
|
||||
return page
|
||||
? {
|
||||
key: page.key,
|
||||
path: page.path,
|
||||
scope: page.scope,
|
||||
summary: page.summary,
|
||||
score: fused.score,
|
||||
matchReasons: fused.matchReasons as WikiSearchMatchReason[],
|
||||
lanes: result.lanes,
|
||||
}
|
||||
: null;
|
||||
})
|
||||
.filter((result): result is LocalKnowledgeSearchResult => result !== null);
|
||||
}
|
||||
|
||||
async function searchLocalKnowledgePagesWithScan(
|
||||
project: KloLocalProject,
|
||||
input: { query: string; userId?: string; limit?: number },
|
||||
): Promise<LocalKnowledgeSearchResult[]> {
|
||||
const terms = input.query
|
||||
.toLowerCase()
|
||||
.split(/\s+/)
|
||||
.map((term) => term.trim())
|
||||
.filter(Boolean);
|
||||
const pages = await loadAllKnowledgePages(project, { userId: input.userId });
|
||||
const results: LocalKnowledgeSearchResult[] = [];
|
||||
for (const page of pages) {
|
||||
const score = scorePage(page, terms);
|
||||
if (score > 0) {
|
||||
results.push({
|
||||
key: page.key,
|
||||
path: page.path,
|
||||
scope: page.scope,
|
||||
summary: page.summary,
|
||||
score,
|
||||
matchReasons: ['token' as const],
|
||||
});
|
||||
}
|
||||
}
|
||||
return results
|
||||
.sort((left, right) => right.score - left.score || left.path.localeCompare(right.path))
|
||||
.slice(0, input.limit ?? results.length);
|
||||
}
|
||||
|
||||
export async function searchLocalKnowledgePages(
|
||||
project: KloLocalProject,
|
||||
input: { query: string; userId?: string; embeddingService?: KloEmbeddingPort | null; limit?: number },
|
||||
): Promise<LocalKnowledgeSearchResult[]> {
|
||||
if (project.config.storage.search === 'sqlite-fts5') {
|
||||
return searchLocalKnowledgePagesWithSqlite(project, input);
|
||||
}
|
||||
return searchLocalKnowledgePagesWithScan(project, input);
|
||||
}
|
||||
68
packages/context/src/wiki/ports.ts
Normal file
68
packages/context/src/wiki/ports.ts
Normal file
|
|
@ -0,0 +1,68 @@
|
|||
import type { KloFileStorePort } from '../core/file-store.js';
|
||||
|
||||
export interface UpsertPageParams {
|
||||
scope: string;
|
||||
scopeId: string | null;
|
||||
pageKey: string;
|
||||
summary: string;
|
||||
usageMode: string;
|
||||
sortOrder: number;
|
||||
searchText: string;
|
||||
embedding: number[] | null;
|
||||
contentHash?: string | null;
|
||||
sourceRunId?: string | null;
|
||||
}
|
||||
|
||||
export interface KnowledgeIndexPort {
|
||||
upsertPage(params: UpsertPageParams): Promise<void>;
|
||||
applyDiffTransactional(params: {
|
||||
runId: string;
|
||||
upserts: UpsertPageParams[];
|
||||
deletes: Array<{ scope: string; scopeId: string | null; pageKey: string }>;
|
||||
}): Promise<void>;
|
||||
getExistingSearchTexts(
|
||||
scope: string,
|
||||
scopeId: string | null,
|
||||
): Promise<Map<string, { searchText: string; hasEmbedding: boolean }>>;
|
||||
deleteStale(scope: string, scopeId: string | null, keepKeys: string[]): Promise<void>;
|
||||
deleteByScope(scope: string, scopeId: string | null): Promise<void>;
|
||||
deleteByKey(scope: string, scopeId: string | null, pageKey: string): Promise<void>;
|
||||
findPageByKey(
|
||||
scope: string,
|
||||
scopeId: string | null,
|
||||
pageKey: string,
|
||||
): Promise<{ id?: string; page_key: string } | null | undefined>;
|
||||
listPagesForUser(
|
||||
userId: string,
|
||||
): Promise<Array<{ id?: string; page_key: string; summary: string; scope: string; scope_id: string | null }>>;
|
||||
getUserPageCount(userId: string): Promise<number>;
|
||||
incrementUsageCount(pageIds: string[]): Promise<void>;
|
||||
searchRRF(
|
||||
userId: string,
|
||||
queryEmbedding: number[] | null,
|
||||
queryText: string,
|
||||
limit: number,
|
||||
): Promise<Array<{ pageKey: string; summary: string; rrfScore: number }>>;
|
||||
}
|
||||
|
||||
export interface KnowledgeEventPort {
|
||||
createEvent(params: {
|
||||
blockId: string | null;
|
||||
eventType: string;
|
||||
actorId: string;
|
||||
chatId?: string | null;
|
||||
messageId?: string | null;
|
||||
payload: Record<string, unknown>;
|
||||
}): Promise<unknown>;
|
||||
}
|
||||
|
||||
export interface KnowledgeGitDiffPort {
|
||||
diffNameStatus(
|
||||
fromSha: string,
|
||||
toSha: string,
|
||||
pathPrefix?: string,
|
||||
): Promise<Array<{ status: string; path: string }>>;
|
||||
getFileAtCommit(path: string, sha: string): Promise<string>;
|
||||
}
|
||||
|
||||
export type WikiFileStorePort = KloFileStorePort<WikiFileStorePort>;
|
||||
115
packages/context/src/wiki/sqlite-knowledge-index.test.ts
Normal file
115
packages/context/src/wiki/sqlite-knowledge-index.test.ts
Normal file
|
|
@ -0,0 +1,115 @@
|
|||
import { access, mkdtemp, rm } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
import { SqliteKnowledgeIndex, type SqliteKnowledgeIndexPage } from './sqlite-knowledge-index.js';
|
||||
|
||||
describe('SqliteKnowledgeIndex', () => {
|
||||
let tempDir: string;
|
||||
let dbPath: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'klo-sqlite-knowledge-index-'));
|
||||
dbPath = join(tempDir, 'db.sqlite');
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function page(overrides: Partial<SqliteKnowledgeIndexPage> = {}): SqliteKnowledgeIndexPage {
|
||||
return {
|
||||
path: 'knowledge/global/revenue.md',
|
||||
key: 'revenue',
|
||||
scope: 'GLOBAL',
|
||||
summary: 'Revenue definition',
|
||||
content: 'Revenue is the sum of paid order amounts.',
|
||||
tags: ['finance'],
|
||||
embedding: null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
it('creates a SQLite FTS5 index and returns lexical lane candidates', async () => {
|
||||
const index = new SqliteKnowledgeIndex({ dbPath });
|
||||
|
||||
index.sync([
|
||||
page(),
|
||||
page({
|
||||
path: 'knowledge/global/support.md',
|
||||
key: 'support',
|
||||
summary: 'Support queue',
|
||||
content: 'Tickets are grouped by priority.',
|
||||
tags: ['operations'],
|
||||
}),
|
||||
]);
|
||||
|
||||
await expect(access(dbPath)).resolves.toBeUndefined();
|
||||
expect(index.searchLexicalCandidates({ queryText: 'paid order', limit: 10 })).toEqual([
|
||||
expect.objectContaining({
|
||||
id: 'knowledge/global/revenue.md',
|
||||
path: 'knowledge/global/revenue.md',
|
||||
rank: 1,
|
||||
rawScore: expect.any(Number),
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it('removes stale rows when the Markdown source list changes', () => {
|
||||
const index = new SqliteKnowledgeIndex({ dbPath });
|
||||
index.rebuild([page(), page({ path: 'knowledge/global/churn.md', key: 'churn', content: 'Churn risk.' })]);
|
||||
expect(index.search('churn', 10)).toHaveLength(1);
|
||||
|
||||
index.rebuild([page()]);
|
||||
|
||||
expect(index.search('churn', 10)).toEqual([]);
|
||||
});
|
||||
|
||||
it('exposes existing search text and embedding state for incremental refresh', () => {
|
||||
const index = new SqliteKnowledgeIndex({ dbPath });
|
||||
index.sync([page({ path: 'knowledge/global/revenue.md', key: 'revenue', embedding: [1, 0] })]);
|
||||
|
||||
expect(index.getExistingPages()).toEqual(
|
||||
new Map([
|
||||
[
|
||||
'knowledge/global/revenue.md',
|
||||
expect.objectContaining({
|
||||
searchText: expect.stringContaining('Revenue definition'),
|
||||
embedding: [1, 0],
|
||||
}),
|
||||
],
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it('returns semantic lane candidates from stored page embeddings', () => {
|
||||
const index = new SqliteKnowledgeIndex({ dbPath });
|
||||
index.sync([
|
||||
page({ path: 'knowledge/global/revenue.md', key: 'revenue', embedding: [1, 0] }),
|
||||
page({ path: 'knowledge/global/support.md', key: 'support', summary: 'Support queue', embedding: [0, 1] }),
|
||||
]);
|
||||
|
||||
expect(index.searchSemanticCandidates({ queryEmbedding: [1, 0], limit: 10 })).toEqual([
|
||||
expect.objectContaining({
|
||||
id: 'knowledge/global/revenue.md',
|
||||
path: 'knowledge/global/revenue.md',
|
||||
rank: 1,
|
||||
rawScore: 1,
|
||||
}),
|
||||
expect.objectContaining({
|
||||
id: 'knowledge/global/support.md',
|
||||
path: 'knowledge/global/support.md',
|
||||
rank: 2,
|
||||
rawScore: 0,
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it('returns an empty result for blank or punctuation-only queries', () => {
|
||||
const index = new SqliteKnowledgeIndex({ dbPath });
|
||||
index.rebuild([page()]);
|
||||
|
||||
expect(index.search(' ', 10)).toEqual([]);
|
||||
expect(index.search('---', 10)).toEqual([]);
|
||||
});
|
||||
});
|
||||
276
packages/context/src/wiki/sqlite-knowledge-index.ts
Normal file
276
packages/context/src/wiki/sqlite-knowledge-index.ts
Normal file
|
|
@ -0,0 +1,276 @@
|
|||
import { mkdirSync } from 'node:fs';
|
||||
import { dirname } from 'node:path';
|
||||
import Database from 'better-sqlite3';
|
||||
import { buildKnowledgeSearchText } from './knowledge-search-text.js';
|
||||
import type { LocalKnowledgeScope } from './local-knowledge.js';
|
||||
|
||||
export interface SqliteKnowledgeIndexOptions {
|
||||
dbPath: string;
|
||||
}
|
||||
|
||||
export interface SqliteKnowledgeIndexPage {
|
||||
path: string;
|
||||
key: string;
|
||||
scope: LocalKnowledgeScope;
|
||||
summary: string;
|
||||
content: string;
|
||||
tags: string[];
|
||||
embedding?: number[] | null;
|
||||
}
|
||||
|
||||
export interface SqliteKnowledgeIndexSearchResult {
|
||||
path: string;
|
||||
score: number;
|
||||
}
|
||||
|
||||
export interface WikiSqliteLaneCandidate {
|
||||
id: string;
|
||||
path: string;
|
||||
rank: number;
|
||||
rawScore: number;
|
||||
}
|
||||
|
||||
export interface ExistingKnowledgeIndexPage {
|
||||
searchText: string;
|
||||
embedding: number[] | null;
|
||||
}
|
||||
|
||||
interface SearchRow {
|
||||
path: string;
|
||||
rank: number;
|
||||
}
|
||||
|
||||
type IndexedPageRow = {
|
||||
path: string;
|
||||
embedding_json: string | null;
|
||||
};
|
||||
|
||||
function cosineSimilarity(left: number[], right: number[]): number {
|
||||
if (left.length === 0 || left.length !== right.length) {
|
||||
return 0;
|
||||
}
|
||||
let dot = 0;
|
||||
let leftNorm = 0;
|
||||
let rightNorm = 0;
|
||||
for (let i = 0; i < left.length; i++) {
|
||||
const l = left[i] ?? 0;
|
||||
const r = right[i] ?? 0;
|
||||
dot += l * r;
|
||||
leftNorm += l * l;
|
||||
rightNorm += r * r;
|
||||
}
|
||||
if (leftNorm === 0 || rightNorm === 0) {
|
||||
return 0;
|
||||
}
|
||||
return dot / (Math.sqrt(leftNorm) * Math.sqrt(rightNorm));
|
||||
}
|
||||
|
||||
function scoreFromRank(rank: number): number {
|
||||
return Number((1 / (1 + Math.abs(rank))).toFixed(6));
|
||||
}
|
||||
|
||||
function parseEmbedding(raw: string | null): number[] | null {
|
||||
if (!raw) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
const embedding = JSON.parse(raw) as unknown;
|
||||
return Array.isArray(embedding) && embedding.every((value) => typeof value === 'number') ? embedding : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeFtsQuery(query: string): string {
|
||||
const terms = query
|
||||
.toLowerCase()
|
||||
.split(/[^a-z0-9_]+/u)
|
||||
.map((term) => term.trim())
|
||||
.filter(Boolean);
|
||||
|
||||
return [...new Set(terms)].map((term) => `"${term.replaceAll('"', '""')}"`).join(' OR ');
|
||||
}
|
||||
|
||||
export class SqliteKnowledgeIndex {
|
||||
private readonly db: Database.Database;
|
||||
|
||||
constructor(options: SqliteKnowledgeIndexOptions) {
|
||||
mkdirSync(dirname(options.dbPath), { recursive: true });
|
||||
this.db = new Database(options.dbPath);
|
||||
this.db.pragma('journal_mode = WAL');
|
||||
this.db.pragma('foreign_keys = ON');
|
||||
this.db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS knowledge_pages (
|
||||
path TEXT PRIMARY KEY,
|
||||
key TEXT NOT NULL,
|
||||
scope TEXT NOT NULL,
|
||||
summary TEXT NOT NULL,
|
||||
content TEXT NOT NULL,
|
||||
tags TEXT NOT NULL,
|
||||
search_text TEXT NOT NULL,
|
||||
embedding_json TEXT
|
||||
);
|
||||
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS knowledge_pages_fts USING fts5(
|
||||
path UNINDEXED,
|
||||
key,
|
||||
summary,
|
||||
content,
|
||||
tags
|
||||
);
|
||||
`);
|
||||
const columns = this.db.prepare('PRAGMA table_info(knowledge_pages)').all() as Array<{ name: string }>;
|
||||
const columnNames = new Set(columns.map((column) => column.name));
|
||||
if (!columnNames.has('search_text')) {
|
||||
this.db.exec("ALTER TABLE knowledge_pages ADD COLUMN search_text TEXT NOT NULL DEFAULT ''");
|
||||
}
|
||||
if (!columnNames.has('embedding_json')) {
|
||||
this.db.exec('ALTER TABLE knowledge_pages ADD COLUMN embedding_json TEXT');
|
||||
}
|
||||
}
|
||||
|
||||
sync(pages: SqliteKnowledgeIndexPage[]): void {
|
||||
const keepPaths = pages.map((page) => page.path);
|
||||
const clearPages =
|
||||
keepPaths.length === 0
|
||||
? this.db.prepare('DELETE FROM knowledge_pages')
|
||||
: this.db.prepare(`DELETE FROM knowledge_pages WHERE path NOT IN (${keepPaths.map(() => '?').join(', ')})`);
|
||||
const clearFts =
|
||||
keepPaths.length === 0
|
||||
? this.db.prepare('DELETE FROM knowledge_pages_fts')
|
||||
: this.db.prepare(`DELETE FROM knowledge_pages_fts WHERE path NOT IN (${keepPaths.map(() => '?').join(', ')})`);
|
||||
const upsertPage = this.db.prepare(`
|
||||
INSERT INTO knowledge_pages (path, key, scope, summary, content, tags, search_text, embedding_json)
|
||||
VALUES (@path, @key, @scope, @summary, @content, @tags, @searchText, @embeddingJson)
|
||||
ON CONFLICT(path) DO UPDATE SET
|
||||
key = excluded.key,
|
||||
scope = excluded.scope,
|
||||
summary = excluded.summary,
|
||||
content = excluded.content,
|
||||
tags = excluded.tags,
|
||||
search_text = excluded.search_text,
|
||||
embedding_json = excluded.embedding_json
|
||||
`);
|
||||
const deleteFts = this.db.prepare('DELETE FROM knowledge_pages_fts WHERE path = @path');
|
||||
const insertFts = this.db.prepare(`
|
||||
INSERT INTO knowledge_pages_fts (path, key, summary, content, tags)
|
||||
VALUES (@path, @key, @summary, @content, @tags)
|
||||
`);
|
||||
|
||||
const transaction = this.db.transaction((items: SqliteKnowledgeIndexPage[]) => {
|
||||
clearPages.run(...keepPaths);
|
||||
clearFts.run(...keepPaths);
|
||||
for (const page of items) {
|
||||
const searchText = buildKnowledgeSearchText(page.key, page.summary, page.content, page.tags);
|
||||
const row = {
|
||||
path: page.path,
|
||||
key: page.key,
|
||||
scope: page.scope,
|
||||
summary: page.summary,
|
||||
content: searchText,
|
||||
tags: page.tags.join(' '),
|
||||
searchText,
|
||||
embeddingJson: page.embedding ? JSON.stringify(page.embedding) : null,
|
||||
};
|
||||
upsertPage.run(row);
|
||||
deleteFts.run(row);
|
||||
insertFts.run(row);
|
||||
}
|
||||
});
|
||||
|
||||
transaction(pages);
|
||||
}
|
||||
|
||||
rebuild(pages: SqliteKnowledgeIndexPage[]): void {
|
||||
this.sync(pages);
|
||||
}
|
||||
|
||||
getExistingPages(): Map<string, ExistingKnowledgeIndexPage> {
|
||||
const rows = this.db
|
||||
.prepare(
|
||||
`
|
||||
SELECT path, search_text, embedding_json
|
||||
FROM knowledge_pages
|
||||
ORDER BY path ASC
|
||||
`,
|
||||
)
|
||||
.all() as Array<{ path: string; search_text: string; embedding_json: string | null }>;
|
||||
|
||||
return new Map(
|
||||
rows.map((row) => [
|
||||
row.path,
|
||||
{
|
||||
searchText: row.search_text,
|
||||
embedding: parseEmbedding(row.embedding_json),
|
||||
},
|
||||
]),
|
||||
);
|
||||
}
|
||||
|
||||
searchLexicalCandidates(input: { queryText: string; limit: number }): WikiSqliteLaneCandidate[] {
|
||||
const ftsQuery = normalizeFtsQuery(input.queryText);
|
||||
if (!ftsQuery) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const rows = this.db
|
||||
.prepare(
|
||||
`
|
||||
SELECT path, bm25(knowledge_pages_fts) AS rank
|
||||
FROM knowledge_pages_fts
|
||||
WHERE knowledge_pages_fts MATCH ?
|
||||
ORDER BY rank ASC, path ASC
|
||||
LIMIT ?
|
||||
`,
|
||||
)
|
||||
.all(ftsQuery, Math.max(1, input.limit)) as SearchRow[];
|
||||
|
||||
return rows.map((row, index) => ({
|
||||
id: row.path,
|
||||
path: row.path,
|
||||
rank: index + 1,
|
||||
rawScore: Number(row.rank),
|
||||
}));
|
||||
}
|
||||
|
||||
searchSemanticCandidates(input: { queryEmbedding: number[]; limit: number }): WikiSqliteLaneCandidate[] {
|
||||
const rows = this.db
|
||||
.prepare(
|
||||
`
|
||||
SELECT path, embedding_json
|
||||
FROM knowledge_pages
|
||||
ORDER BY path ASC
|
||||
`,
|
||||
)
|
||||
.all() as IndexedPageRow[];
|
||||
|
||||
return rows
|
||||
.flatMap((row) => {
|
||||
if (!row.embedding_json) {
|
||||
return [];
|
||||
}
|
||||
const embedding = parseEmbedding(row.embedding_json);
|
||||
if (!embedding) {
|
||||
return [];
|
||||
}
|
||||
return [
|
||||
{
|
||||
id: row.path,
|
||||
path: row.path,
|
||||
rank: 0,
|
||||
rawScore: cosineSimilarity(input.queryEmbedding, embedding),
|
||||
},
|
||||
];
|
||||
})
|
||||
.sort((left, right) => right.rawScore - left.rawScore || left.path.localeCompare(right.path))
|
||||
.slice(0, Math.max(1, input.limit))
|
||||
.map((candidate, index) => ({ ...candidate, rank: index + 1 }));
|
||||
}
|
||||
|
||||
search(query: string, limit: number): SqliteKnowledgeIndexSearchResult[] {
|
||||
return this.searchLexicalCandidates({ queryText: query, limit }).map((row) => ({
|
||||
path: row.path,
|
||||
score: scoreFromRank(row.rawScore),
|
||||
}));
|
||||
}
|
||||
}
|
||||
5
packages/context/src/wiki/tools/index.ts
Normal file
5
packages/context/src/wiki/tools/index.ts
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
export { WikiListTagsTool } from './wiki-list-tags.tool.js';
|
||||
export { WikiReadTool } from './wiki-read.tool.js';
|
||||
export { WikiRemoveTool } from './wiki-remove.tool.js';
|
||||
export { WikiSearchTool } from './wiki-search.tool.js';
|
||||
export { WikiWriteTool } from './wiki-write.tool.js';
|
||||
42
packages/context/src/wiki/tools/wiki-list-tags.tool.test.ts
Normal file
42
packages/context/src/wiki/tools/wiki-list-tags.tool.test.ts
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
import { describe, expect, it, vi } from 'vitest';
|
||||
import type { ToolContext } from '../../tools/index.js';
|
||||
import { WikiListTagsTool } from './wiki-list-tags.tool.js';
|
||||
|
||||
describe('WikiListTagsTool', () => {
|
||||
const baseContext: ToolContext = { sourceId: 's', messageId: 'm', userId: 'u' };
|
||||
|
||||
it("returns distinct sorted tags across the user's visible pages", async () => {
|
||||
const pagesRepository = {
|
||||
listPagesForUser: vi.fn().mockResolvedValue([
|
||||
{ scope: 'GLOBAL', scope_id: null, page_key: 'k1' },
|
||||
{ scope: 'USER', scope_id: 'u', page_key: 'k2' },
|
||||
]),
|
||||
};
|
||||
const wikiService = {
|
||||
readPage: vi.fn().mockImplementation((_scope, _scopeId, key) => {
|
||||
if (key === 'k1') {
|
||||
return Promise.resolve({ frontmatter: { tags: ['metrics', 'finance'] }, content: '' });
|
||||
}
|
||||
if (key === 'k2') {
|
||||
return Promise.resolve({ frontmatter: { tags: ['metrics'] }, content: '' });
|
||||
}
|
||||
return Promise.resolve(null);
|
||||
}),
|
||||
};
|
||||
const tool = new WikiListTagsTool(wikiService as any, pagesRepository as any);
|
||||
|
||||
const result = await tool.call({}, baseContext);
|
||||
expect(result.markdown).toContain('finance');
|
||||
expect(result.markdown).toContain('metrics');
|
||||
expect(result.structured.tags).toEqual(['finance', 'metrics']);
|
||||
});
|
||||
|
||||
it('returns a friendly message when no pages have tags', async () => {
|
||||
const pagesRepository = { listPagesForUser: vi.fn().mockResolvedValue([]) };
|
||||
const wikiService = { readPage: vi.fn() };
|
||||
const tool = new WikiListTagsTool(wikiService as any, pagesRepository as any);
|
||||
|
||||
const result = await tool.call({}, baseContext);
|
||||
expect(result.markdown).toMatch(/no tags/i);
|
||||
});
|
||||
});
|
||||
49
packages/context/src/wiki/tools/wiki-list-tags.tool.ts
Normal file
49
packages/context/src/wiki/tools/wiki-list-tags.tool.ts
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
import { z } from 'zod';
|
||||
import type { KnowledgeIndexPort } from '../ports.js';
|
||||
type BlockScope = 'GLOBAL' | 'USER';
|
||||
import { KnowledgeWikiService } from '../index.js';
|
||||
import { BaseTool, type ToolContext, type ToolOutput } from '../../tools/index.js';
|
||||
|
||||
const wikiListTagsInputSchema = z.object({});
|
||||
|
||||
type WikiListTagsInput = z.infer<typeof wikiListTagsInputSchema>;
|
||||
|
||||
export class WikiListTagsTool extends BaseTool<typeof wikiListTagsInputSchema> {
|
||||
readonly name = 'wiki_list_tags';
|
||||
|
||||
constructor(
|
||||
private readonly wikiService: KnowledgeWikiService,
|
||||
private readonly pagesRepository: KnowledgeIndexPort,
|
||||
) {
|
||||
super();
|
||||
}
|
||||
|
||||
get description(): string {
|
||||
return `<purpose>
|
||||
List distinct topic tags across all wiki pages visible to the user.
|
||||
Call before writing a new page so you can reuse existing tags consistently instead of coining near-duplicates.
|
||||
</purpose>`;
|
||||
}
|
||||
|
||||
get inputSchema() {
|
||||
return wikiListTagsInputSchema;
|
||||
}
|
||||
|
||||
async call(_input: WikiListTagsInput, context: ToolContext): Promise<ToolOutput<{ tags: string[] }>> {
|
||||
const pages = await this.pagesRepository.listPagesForUser(context.userId);
|
||||
const set = new Set<string>();
|
||||
for (const p of pages) {
|
||||
const scope = p.scope as BlockScope;
|
||||
const scopeId = scope === 'USER' ? p.scope_id : null;
|
||||
const page = await this.wikiService.readPage(scope, scopeId, p.page_key);
|
||||
for (const t of page?.frontmatter.tags ?? []) {
|
||||
set.add(t);
|
||||
}
|
||||
}
|
||||
const tags = [...set].sort();
|
||||
return {
|
||||
markdown: tags.length === 0 ? '(no tags in use yet)' : tags.join(', '),
|
||||
structured: { tags },
|
||||
};
|
||||
}
|
||||
}
|
||||
82
packages/context/src/wiki/tools/wiki-read.tool.ts
Normal file
82
packages/context/src/wiki/tools/wiki-read.tool.ts
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
import { z } from 'zod';
|
||||
import type { KnowledgeIndexPort } from '../ports.js';
|
||||
import { KnowledgeWikiService } from '../index.js';
|
||||
import { BaseTool, type ToolContext, type ToolOutput } from '../../tools/index.js';
|
||||
|
||||
const WikiReadInputSchema = z.object({
|
||||
key: z
|
||||
.string()
|
||||
.describe('The block_key to read. Check the <knowledge_index> in the system prompt for available keys.'),
|
||||
});
|
||||
|
||||
type WikiReadInput = z.infer<typeof WikiReadInputSchema>;
|
||||
|
||||
interface WikiReadStructured {
|
||||
blockKey: string;
|
||||
content: string;
|
||||
scope: string;
|
||||
found: boolean;
|
||||
tags?: string[];
|
||||
refs?: string[];
|
||||
}
|
||||
|
||||
export class WikiReadTool extends BaseTool<typeof WikiReadInputSchema> {
|
||||
readonly name = 'wiki_read';
|
||||
|
||||
constructor(
|
||||
private readonly wikiService: KnowledgeWikiService,
|
||||
private readonly pagesRepository: KnowledgeIndexPort,
|
||||
) {
|
||||
super();
|
||||
}
|
||||
|
||||
get description(): string {
|
||||
return (
|
||||
'Load the full content of a knowledge block by its key. ' +
|
||||
'Use this to retrieve detailed rules, preferences, or definitions listed in the <knowledge_index>. ' +
|
||||
'Call this when the user query relates to a topic covered by an available knowledge block.'
|
||||
);
|
||||
}
|
||||
|
||||
get inputSchema() {
|
||||
return WikiReadInputSchema;
|
||||
}
|
||||
|
||||
async call(input: WikiReadInput, context: ToolContext): Promise<ToolOutput<WikiReadStructured>> {
|
||||
const page = await this.wikiService.readPageForUser(context.userId, input.key);
|
||||
|
||||
if (!page) {
|
||||
return {
|
||||
markdown: `No knowledge block found with key "${input.key}".`,
|
||||
structured: { blockKey: input.key, content: '', scope: '', found: false },
|
||||
};
|
||||
}
|
||||
|
||||
const indexEntry = await this.pagesRepository.findPageByKey(
|
||||
page.scope,
|
||||
page.scope === 'USER' ? context.userId : null,
|
||||
input.key,
|
||||
);
|
||||
if (indexEntry?.id) {
|
||||
void this.pagesRepository.incrementUsageCount([indexEntry.id]);
|
||||
}
|
||||
|
||||
let md = `## ${page.pageKey}\n\n${page.content}`;
|
||||
const refs = page.frontmatter.refs;
|
||||
if (refs && refs.length > 0) {
|
||||
md += `\n\nSee also: ${refs.map((r) => `[[${r}]]`).join(', ')}`;
|
||||
}
|
||||
|
||||
return {
|
||||
markdown: md,
|
||||
structured: {
|
||||
blockKey: page.pageKey,
|
||||
content: page.content,
|
||||
scope: page.scope,
|
||||
found: true,
|
||||
tags: page.frontmatter.tags,
|
||||
refs: page.frontmatter.refs,
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
59
packages/context/src/wiki/tools/wiki-remove.tool.test.ts
Normal file
59
packages/context/src/wiki/tools/wiki-remove.tool.test.ts
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
import { describe, expect, it, vi } from 'vitest';
|
||||
import type { ToolSession } from '../../tools/index.js';
|
||||
import { createTouchedSlSources, type ToolContext } from '../../tools/index.js';
|
||||
import { WikiRemoveTool } from './wiki-remove.tool.js';
|
||||
|
||||
describe('WikiRemoveTool', () => {
|
||||
const baseContext: ToolContext = { sourceId: 's', messageId: 'm', userId: 'u' };
|
||||
|
||||
it('removes an existing page when no session is present', async () => {
|
||||
const wikiService = {
|
||||
deletePage: vi.fn().mockResolvedValue(undefined),
|
||||
deleteFromIndex: vi.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
const pagesRepository = {
|
||||
findPageByKey: vi.fn().mockResolvedValue({ page_key: 'old' }),
|
||||
};
|
||||
const knowledgeRepository = { createEvent: vi.fn().mockResolvedValue(undefined) };
|
||||
const tool = new WikiRemoveTool(wikiService as any, pagesRepository as any, knowledgeRepository as any);
|
||||
const result = await tool.call({ key: 'old' } as any, baseContext);
|
||||
expect(wikiService.deletePage).toHaveBeenCalledTimes(1);
|
||||
expect(wikiService.deleteFromIndex).toHaveBeenCalledTimes(1);
|
||||
expect(result.markdown).toMatch(/removed/i);
|
||||
});
|
||||
|
||||
it('skips deleteFromIndex when session is worktree-scoped', async () => {
|
||||
const wikiService = {
|
||||
deletePage: vi.fn().mockResolvedValue(undefined),
|
||||
deleteFromIndex: vi.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
const pagesRepository = { findPageByKey: vi.fn().mockResolvedValue({ page_key: 'old' }) };
|
||||
const knowledgeRepository = { createEvent: vi.fn().mockResolvedValue(undefined) };
|
||||
const tool = new WikiRemoveTool(wikiService as any, pagesRepository as any, knowledgeRepository as any);
|
||||
const session: ToolSession = {
|
||||
connectionId: 'c',
|
||||
isWorktreeScoped: true,
|
||||
preHead: null,
|
||||
touchedSlSources: createTouchedSlSources(),
|
||||
actions: [],
|
||||
semanticLayerService: {} as any,
|
||||
wikiService: wikiService as any,
|
||||
configService: {} as any,
|
||||
gitService: {} as any,
|
||||
};
|
||||
await tool.call({ key: 'old' } as any, { ...baseContext, session });
|
||||
expect(wikiService.deletePage).toHaveBeenCalledTimes(1);
|
||||
expect(wikiService.deleteFromIndex).not.toHaveBeenCalled();
|
||||
expect(session.actions).toContainEqual(expect.objectContaining({ target: 'wiki', type: 'removed', key: 'old' }));
|
||||
});
|
||||
|
||||
it('returns a friendly message when the page does not exist', async () => {
|
||||
const wikiService = { deletePage: vi.fn(), deleteFromIndex: vi.fn() };
|
||||
const pagesRepository = { findPageByKey: vi.fn().mockResolvedValue(null) };
|
||||
const knowledgeRepository = { createEvent: vi.fn() };
|
||||
const tool = new WikiRemoveTool(wikiService as any, pagesRepository as any, knowledgeRepository as any);
|
||||
const result = await tool.call({ key: 'missing' } as any, baseContext);
|
||||
expect(result.structured.success).toBe(false);
|
||||
expect(result.markdown).toMatch(/not found/i);
|
||||
});
|
||||
});
|
||||
85
packages/context/src/wiki/tools/wiki-remove.tool.ts
Normal file
85
packages/context/src/wiki/tools/wiki-remove.tool.ts
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
import { z } from 'zod';
|
||||
import type { KnowledgeIndexPort } from '../ports.js';
|
||||
import type { KnowledgeEventPort } from '../ports.js';
|
||||
type BlockScope = 'GLOBAL' | 'USER';
|
||||
import { KnowledgeWikiService } from '../index.js';
|
||||
import { BaseTool, type ToolContext, type ToolOutput } from '../../tools/index.js';
|
||||
|
||||
const SYSTEM_AUTHOR = 'System User';
|
||||
const SYSTEM_EMAIL = 'system@example.com';
|
||||
|
||||
const wikiRemoveInputSchema = z.object({
|
||||
key: z.string().describe('The page key to remove'),
|
||||
});
|
||||
|
||||
type WikiRemoveInput = z.infer<typeof wikiRemoveInputSchema>;
|
||||
|
||||
interface WikiRemoveStructured {
|
||||
success: boolean;
|
||||
key: string;
|
||||
}
|
||||
|
||||
export class WikiRemoveTool extends BaseTool<typeof wikiRemoveInputSchema> {
|
||||
readonly name = 'wiki_remove';
|
||||
|
||||
constructor(
|
||||
private readonly wikiService: KnowledgeWikiService,
|
||||
private readonly pagesRepository: KnowledgeIndexPort,
|
||||
private readonly knowledgeRepository: KnowledgeEventPort,
|
||||
) {
|
||||
super();
|
||||
}
|
||||
|
||||
get description(): string {
|
||||
return `<purpose>Remove a knowledge page that is no longer relevant.</purpose>`;
|
||||
}
|
||||
|
||||
get inputSchema() {
|
||||
return wikiRemoveInputSchema;
|
||||
}
|
||||
|
||||
async call(input: WikiRemoveInput, context: ToolContext): Promise<ToolOutput<WikiRemoveStructured>> {
|
||||
const wikiService = context.session?.wikiService ?? this.wikiService;
|
||||
const writesGlobal = !!context.session;
|
||||
const skipIndex = context.session?.isWorktreeScoped === true;
|
||||
|
||||
const scope: BlockScope = writesGlobal ? 'GLOBAL' : 'USER';
|
||||
const scopeId = scope === 'USER' ? context.userId : null;
|
||||
|
||||
const existing = await this.pagesRepository.findPageByKey(scope, scopeId, input.key);
|
||||
if (!existing) {
|
||||
return {
|
||||
markdown: `Page "${input.key}" not found.`,
|
||||
structured: { success: false, key: input.key },
|
||||
};
|
||||
}
|
||||
|
||||
await wikiService.deletePage(scope, scopeId, input.key, SYSTEM_AUTHOR, SYSTEM_EMAIL);
|
||||
if (!skipIndex) {
|
||||
await wikiService.deleteFromIndex(scope, scopeId, input.key);
|
||||
}
|
||||
|
||||
await this.knowledgeRepository.createEvent({
|
||||
blockId: null,
|
||||
eventType: 'BLOCK_REMOVED',
|
||||
actorId: context.userId,
|
||||
chatId: null,
|
||||
messageId: null,
|
||||
payload: { removedKey: input.key, blockKey: input.key },
|
||||
});
|
||||
|
||||
if (context.session) {
|
||||
context.session.actions.push({
|
||||
target: 'wiki',
|
||||
type: 'removed',
|
||||
key: input.key,
|
||||
detail: `Removed page "${input.key}"`,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
markdown: `Page "${input.key}" removed.`,
|
||||
structured: { success: true, key: input.key },
|
||||
};
|
||||
}
|
||||
}
|
||||
41
packages/context/src/wiki/tools/wiki-search.tool.test.ts
Normal file
41
packages/context/src/wiki/tools/wiki-search.tool.test.ts
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
import { describe, expect, it, vi } from 'vitest';
|
||||
import { WikiSearchTool } from './wiki-search.tool.js';
|
||||
|
||||
describe('WikiSearchTool', () => {
|
||||
it('searches through the injected wiki adapter port', async () => {
|
||||
const search = vi.fn(async () => ({
|
||||
results: [
|
||||
{
|
||||
key: 'metrics/revenue',
|
||||
path: 'knowledge/global/metrics/revenue.md',
|
||||
scope: 'GLOBAL' as const,
|
||||
summary: 'Revenue metric definition',
|
||||
score: 0.02459016393442623,
|
||||
matchReasons: ['lexical' as const, 'token' as const],
|
||||
},
|
||||
],
|
||||
totalFound: 1,
|
||||
}));
|
||||
const tool = new WikiSearchTool({ search });
|
||||
|
||||
const result = await tool.call(
|
||||
{ query: 'paid order', limit: 5 },
|
||||
{ sourceId: 'test', messageId: 'message-1', userId: 'agent' },
|
||||
);
|
||||
|
||||
expect(search).toHaveBeenCalledWith({ userId: 'agent', query: 'paid order', limit: 5 });
|
||||
expect(result.structured).toEqual({
|
||||
results: [
|
||||
{
|
||||
blockKey: 'metrics/revenue',
|
||||
path: 'knowledge/global/metrics/revenue.md',
|
||||
summary: 'Revenue metric definition',
|
||||
score: 0.02459016393442623,
|
||||
matchReasons: ['lexical', 'token'],
|
||||
},
|
||||
],
|
||||
totalFound: 1,
|
||||
});
|
||||
expect(result.markdown).toContain('**metrics/revenue**');
|
||||
});
|
||||
});
|
||||
92
packages/context/src/wiki/tools/wiki-search.tool.ts
Normal file
92
packages/context/src/wiki/tools/wiki-search.tool.ts
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
import { z } from 'zod';
|
||||
import { BaseTool, type ToolContext, type ToolOutput } from '../../tools/index.js';
|
||||
import type { WikiSearchLaneSummary, WikiSearchMatchReason } from '../types.js';
|
||||
|
||||
const WikiSearchInputSchema = z.object({
|
||||
query: z.string().describe('Natural language search query to find relevant knowledge blocks.'),
|
||||
limit: z.number().optional().default(10).describe('Maximum number of results to return (default 10).'),
|
||||
});
|
||||
|
||||
type WikiSearchInput = z.infer<typeof WikiSearchInputSchema>;
|
||||
|
||||
interface WikiSearchResult {
|
||||
blockKey: string;
|
||||
path: string;
|
||||
summary: string;
|
||||
score: number;
|
||||
matchReasons?: WikiSearchMatchReason[];
|
||||
lanes?: WikiSearchLaneSummary[];
|
||||
}
|
||||
|
||||
interface WikiSearchStructured {
|
||||
results: WikiSearchResult[];
|
||||
totalFound: number;
|
||||
}
|
||||
|
||||
export interface WikiSearchAdapterPort {
|
||||
search(input: { userId: string; query: string; limit: number }): Promise<{
|
||||
results: Array<{
|
||||
key: string;
|
||||
path: string;
|
||||
summary: string;
|
||||
score: number;
|
||||
matchReasons?: WikiSearchMatchReason[];
|
||||
lanes?: WikiSearchLaneSummary[];
|
||||
}>;
|
||||
totalFound: number;
|
||||
}>;
|
||||
}
|
||||
|
||||
export class WikiSearchTool extends BaseTool<typeof WikiSearchInputSchema> {
|
||||
readonly name = 'wiki_search';
|
||||
|
||||
constructor(private readonly searchAdapter: WikiSearchAdapterPort) {
|
||||
super();
|
||||
}
|
||||
|
||||
get description(): string {
|
||||
return (
|
||||
'Search knowledge blocks by hybrid lexical, semantic, and token matching. ' +
|
||||
'Use this when you need to find knowledge on a topic not visible in the discovery index. ' +
|
||||
'Returns ranked summaries — use wiki_read to load the full content of specific results.'
|
||||
);
|
||||
}
|
||||
|
||||
get inputSchema() {
|
||||
return WikiSearchInputSchema;
|
||||
}
|
||||
|
||||
async call(input: WikiSearchInput, context: ToolContext): Promise<ToolOutput<WikiSearchStructured>> {
|
||||
const response = await this.searchAdapter.search({
|
||||
userId: context.userId,
|
||||
query: input.query,
|
||||
limit: input.limit,
|
||||
});
|
||||
|
||||
if (response.results.length === 0) {
|
||||
return {
|
||||
markdown: `No knowledge blocks found matching "${input.query}".`,
|
||||
structured: { results: [], totalFound: 0 },
|
||||
};
|
||||
}
|
||||
|
||||
const lines = response.results.map((r, i) => `${i + 1}. **${r.key}**: ${r.summary}`);
|
||||
|
||||
const structured: WikiSearchStructured = {
|
||||
results: response.results.map((r) => ({
|
||||
blockKey: r.key,
|
||||
path: r.path,
|
||||
summary: r.summary,
|
||||
score: r.score,
|
||||
matchReasons: r.matchReasons,
|
||||
lanes: r.lanes,
|
||||
})),
|
||||
totalFound: response.totalFound,
|
||||
};
|
||||
|
||||
return {
|
||||
markdown: `Found ${response.results.length} knowledge block(s):\n\n${lines.join('\n')}`,
|
||||
structured,
|
||||
};
|
||||
}
|
||||
}
|
||||
168
packages/context/src/wiki/tools/wiki-write.tool.test.ts
Normal file
168
packages/context/src/wiki/tools/wiki-write.tool.test.ts
Normal file
|
|
@ -0,0 +1,168 @@
|
|||
import { describe, expect, it, vi } from 'vitest';
|
||||
import type { ToolSession } from '../../tools/index.js';
|
||||
import { createTouchedSlSources, type ToolContext } from '../../tools/index.js';
|
||||
import { WikiWriteTool } from './wiki-write.tool.js';
|
||||
|
||||
function makeTool(overrides: any = {}) {
|
||||
const wikiService = {
|
||||
readPage: vi.fn().mockResolvedValue(null),
|
||||
writePage: vi.fn().mockResolvedValue(undefined),
|
||||
syncSinglePage: vi.fn().mockResolvedValue(undefined),
|
||||
...overrides.wikiService,
|
||||
};
|
||||
const pagesRepository = {
|
||||
findPageByKey: vi.fn().mockResolvedValue(null),
|
||||
getUserPageCount: vi.fn().mockResolvedValue(0),
|
||||
...overrides.pagesRepository,
|
||||
};
|
||||
const knowledgeRepository = {
|
||||
createEvent: vi.fn().mockResolvedValue(undefined),
|
||||
...overrides.knowledgeRepository,
|
||||
};
|
||||
const tool = new WikiWriteTool(wikiService as any, pagesRepository as any, knowledgeRepository as any);
|
||||
return { tool, wikiService, pagesRepository, knowledgeRepository };
|
||||
}
|
||||
|
||||
describe('WikiWriteTool', () => {
|
||||
const baseContext: ToolContext = { sourceId: 's', messageId: 'm', userId: 'u' };
|
||||
|
||||
it('creates a new page and indexes it when no session is present', async () => {
|
||||
const { tool, wikiService } = makeTool();
|
||||
const result = await tool.call(
|
||||
{ key: 'leads-source', summary: 'Lead source definitions', content: '# Leads' } as any,
|
||||
baseContext,
|
||||
);
|
||||
expect(wikiService.writePage).toHaveBeenCalledTimes(1);
|
||||
expect(wikiService.syncSinglePage).toHaveBeenCalledTimes(1);
|
||||
expect(result.markdown).toMatch(/created/i);
|
||||
});
|
||||
|
||||
it('skips syncSinglePage when session is worktree-scoped', async () => {
|
||||
const { tool, wikiService } = makeTool();
|
||||
const session: ToolSession = {
|
||||
connectionId: 'conn-1',
|
||||
isWorktreeScoped: true,
|
||||
preHead: null,
|
||||
touchedSlSources: createTouchedSlSources(),
|
||||
actions: [],
|
||||
semanticLayerService: {} as any,
|
||||
wikiService: wikiService as any,
|
||||
configService: {} as any,
|
||||
gitService: {} as any,
|
||||
};
|
||||
const context: ToolContext = { ...baseContext, session };
|
||||
await tool.call({ key: 'k', summary: 's', content: '# x' } as any, context);
|
||||
expect(wikiService.writePage).toHaveBeenCalledTimes(1);
|
||||
expect(wikiService.syncSinglePage).not.toHaveBeenCalled();
|
||||
expect(session.actions).toContainEqual(expect.objectContaining({ target: 'wiki', type: 'created', key: 'k' }));
|
||||
});
|
||||
|
||||
it('requires either content or replacements', async () => {
|
||||
const { tool } = makeTool();
|
||||
const result = await tool.call({ key: 'k', summary: 's' } as any, baseContext);
|
||||
expect(result.structured.success).toBe(false);
|
||||
expect(result.markdown).toMatch(/content.*or.*replacements/i);
|
||||
});
|
||||
|
||||
it('writes historic-SQL frontmatter fields', async () => {
|
||||
const { tool, wikiService } = makeTool();
|
||||
|
||||
await tool.call(
|
||||
{
|
||||
key: 'queries/monthly-paid-orders',
|
||||
summary: 'Monthly paid orders',
|
||||
tags: ['historic-sql', 'query-pattern'],
|
||||
sl_refs: ['analytics.orders'],
|
||||
source: 'historic-sql',
|
||||
intent: 'Monthly paid order count',
|
||||
tables: ['analytics.orders'],
|
||||
representative_sql: "SELECT count(*) FROM analytics.orders WHERE status = 'paid'",
|
||||
usage: {
|
||||
executions: 42,
|
||||
distinct_users: 3,
|
||||
first_seen: '2026-02-01',
|
||||
last_seen: '2026-05-04',
|
||||
p50_runtime_ms: 100,
|
||||
p95_runtime_ms: 200,
|
||||
error_rate: 0,
|
||||
rows_produced: 42,
|
||||
},
|
||||
fingerprints: ['fp_paid_orders'],
|
||||
content: '## Monthly paid order count',
|
||||
} as any,
|
||||
baseContext,
|
||||
);
|
||||
|
||||
expect(wikiService.writePage.mock.calls[0][3]).toEqual({
|
||||
summary: 'Monthly paid orders',
|
||||
usage_mode: 'auto',
|
||||
sort_order: 0,
|
||||
tags: ['historic-sql', 'query-pattern'],
|
||||
refs: undefined,
|
||||
sl_refs: ['analytics.orders'],
|
||||
source: 'historic-sql',
|
||||
intent: 'Monthly paid order count',
|
||||
tables: ['analytics.orders'],
|
||||
representative_sql: "SELECT count(*) FROM analytics.orders WHERE status = 'paid'",
|
||||
usage: {
|
||||
executions: 42,
|
||||
distinct_users: 3,
|
||||
first_seen: '2026-02-01',
|
||||
last_seen: '2026-05-04',
|
||||
p50_runtime_ms: 100,
|
||||
p95_runtime_ms: 200,
|
||||
error_rate: 0,
|
||||
rows_produced: 42,
|
||||
},
|
||||
fingerprints: ['fp_paid_orders'],
|
||||
});
|
||||
});
|
||||
|
||||
it('preserves historic-SQL frontmatter fields when update omits them', async () => {
|
||||
const existingFrontmatter = {
|
||||
summary: 'Monthly paid orders',
|
||||
usage_mode: 'auto' as const,
|
||||
sort_order: 0,
|
||||
tags: ['historic-sql'],
|
||||
sl_refs: ['analytics.orders'],
|
||||
source: 'historic-sql',
|
||||
intent: 'Monthly paid order count',
|
||||
tables: ['analytics.orders'],
|
||||
representative_sql: "SELECT count(*) FROM analytics.orders WHERE status = 'paid'",
|
||||
usage: {
|
||||
executions: 42,
|
||||
distinct_users: 3,
|
||||
first_seen: '2026-02-01',
|
||||
last_seen: '2026-05-04',
|
||||
p50_runtime_ms: 100,
|
||||
p95_runtime_ms: 200,
|
||||
error_rate: 0,
|
||||
rows_produced: 42,
|
||||
},
|
||||
fingerprints: ['fp_paid_orders'],
|
||||
};
|
||||
const { tool, wikiService } = makeTool({
|
||||
wikiService: {
|
||||
readPage: vi.fn().mockResolvedValue({
|
||||
pageKey: 'queries/monthly-paid-orders',
|
||||
frontmatter: existingFrontmatter,
|
||||
content: 'old body',
|
||||
}),
|
||||
},
|
||||
});
|
||||
|
||||
await tool.call(
|
||||
{
|
||||
key: 'queries/monthly-paid-orders',
|
||||
summary: 'Monthly paid orders updated',
|
||||
content: '## Monthly paid order count updated',
|
||||
} as any,
|
||||
baseContext,
|
||||
);
|
||||
|
||||
expect(wikiService.writePage.mock.calls[0][3]).toEqual({
|
||||
...existingFrontmatter,
|
||||
summary: 'Monthly paid orders updated',
|
||||
});
|
||||
});
|
||||
});
|
||||
167
packages/context/src/wiki/tools/wiki-write.tool.ts
Normal file
167
packages/context/src/wiki/tools/wiki-write.tool.ts
Normal file
|
|
@ -0,0 +1,167 @@
|
|||
import { z } from 'zod';
|
||||
import type { KnowledgeIndexPort } from '../ports.js';
|
||||
import type { KnowledgeEventPort } from '../ports.js';
|
||||
type BlockScope = 'GLOBAL' | 'USER';
|
||||
import { KnowledgeWikiService, type WikiFrontmatter } from '../index.js';
|
||||
import { applySqlEdits } from '../../tools/sql-edit-replacer.js';
|
||||
import { BaseTool, type ToolContext, type ToolOutput } from '../../tools/index.js';
|
||||
|
||||
const MAX_USER_BLOCKS = 100;
|
||||
const SYSTEM_AUTHOR = 'System User';
|
||||
const SYSTEM_EMAIL = 'system@example.com';
|
||||
|
||||
const historicSqlUsageFrontmatterSchema = z.object({
|
||||
executions: z.number().int().nonnegative(),
|
||||
distinct_users: z.number().int().nonnegative(),
|
||||
first_seen: z.string().min(1),
|
||||
last_seen: z.string().min(1),
|
||||
p50_runtime_ms: z.number().nonnegative().nullable(),
|
||||
p95_runtime_ms: z.number().nonnegative().nullable(),
|
||||
error_rate: z.number().min(0).max(1),
|
||||
rows_produced: z.number().int().nonnegative().optional(),
|
||||
});
|
||||
|
||||
const wikiWriteInputSchema = z.object({
|
||||
key: z.string().max(120),
|
||||
summary: z.string().max(200),
|
||||
content: z.string().max(4000).optional(),
|
||||
replacements: z
|
||||
.array(z.object({ oldText: z.string(), newText: z.string(), reason: z.string().optional() }))
|
||||
.optional(),
|
||||
tags: z.array(z.string()).optional(),
|
||||
refs: z.array(z.string()).optional(),
|
||||
sl_refs: z.array(z.string()).optional(),
|
||||
source: z.string().optional(),
|
||||
intent: z.string().optional(),
|
||||
tables: z.array(z.string()).optional(),
|
||||
representative_sql: z.string().optional(),
|
||||
usage: historicSqlUsageFrontmatterSchema.optional(),
|
||||
fingerprints: z.array(z.string()).optional(),
|
||||
});
|
||||
|
||||
type WikiWriteInput = z.infer<typeof wikiWriteInputSchema>;
|
||||
|
||||
interface WikiWriteStructured {
|
||||
success: boolean;
|
||||
key: string;
|
||||
action?: 'created' | 'updated';
|
||||
}
|
||||
|
||||
export class WikiWriteTool extends BaseTool<typeof wikiWriteInputSchema> {
|
||||
readonly name = 'wiki_write';
|
||||
|
||||
constructor(
|
||||
private readonly wikiService: KnowledgeWikiService,
|
||||
private readonly pagesRepository: KnowledgeIndexPort,
|
||||
private readonly knowledgeRepository: KnowledgeEventPort,
|
||||
) {
|
||||
super();
|
||||
}
|
||||
|
||||
get description(): string {
|
||||
return `<purpose>
|
||||
Create or update a knowledge page. Provide content for create/rewrite, or replacements for targeted edits.
|
||||
tags/refs/sl_refs use REPLACE semantics: omit to keep existing on update, [] to clear, [values] to set.
|
||||
</purpose>`;
|
||||
}
|
||||
|
||||
get inputSchema() {
|
||||
return wikiWriteInputSchema;
|
||||
}
|
||||
|
||||
async call(input: WikiWriteInput, context: ToolContext): Promise<ToolOutput<WikiWriteStructured>> {
|
||||
const wikiService = context.session?.wikiService ?? this.wikiService;
|
||||
const writesGlobal = !!context.session;
|
||||
const skipIndex = context.session?.isWorktreeScoped === true;
|
||||
|
||||
if (!input.content && (!input.replacements || input.replacements.length === 0)) {
|
||||
return {
|
||||
markdown: 'Error: provide either content (for create/rewrite) or replacements (for edits).',
|
||||
structured: { success: false, key: input.key },
|
||||
};
|
||||
}
|
||||
|
||||
const scope: BlockScope = writesGlobal ? 'GLOBAL' : 'USER';
|
||||
const scopeId = scope === 'USER' ? context.userId : null;
|
||||
const existing = await wikiService.readPage(scope, scopeId, input.key);
|
||||
|
||||
if (!existing && !input.content) {
|
||||
return {
|
||||
markdown: `Page "${input.key}" does not exist. Provide content to create it.`,
|
||||
structured: { success: false, key: input.key },
|
||||
};
|
||||
}
|
||||
|
||||
if (scope === 'USER' && !existing) {
|
||||
const count = await this.pagesRepository.getUserPageCount(context.userId);
|
||||
if (count >= MAX_USER_BLOCKS) {
|
||||
return {
|
||||
markdown: `Cannot create "${input.key}": user has reached the limit of ${MAX_USER_BLOCKS} pages.`,
|
||||
structured: { success: false, key: input.key },
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const existingFm = existing?.frontmatter;
|
||||
const resolvedTags = input.tags === undefined ? existingFm?.tags : input.tags;
|
||||
const resolvedRefs = input.refs === undefined ? existingFm?.refs : input.refs;
|
||||
const resolvedSlRefs = input.sl_refs === undefined ? existingFm?.sl_refs : input.sl_refs;
|
||||
|
||||
let finalContent: string;
|
||||
const finalFm: WikiFrontmatter = {
|
||||
summary: input.summary,
|
||||
usage_mode: existingFm?.usage_mode ?? 'auto',
|
||||
sort_order: existingFm?.sort_order ?? 0,
|
||||
tags: resolvedTags,
|
||||
refs: resolvedRefs,
|
||||
sl_refs: resolvedSlRefs,
|
||||
source: input.source === undefined ? existingFm?.source : input.source,
|
||||
intent: input.intent === undefined ? existingFm?.intent : input.intent,
|
||||
tables: input.tables === undefined ? existingFm?.tables : input.tables,
|
||||
representative_sql:
|
||||
input.representative_sql === undefined ? existingFm?.representative_sql : input.representative_sql,
|
||||
usage: input.usage === undefined ? existingFm?.usage : input.usage,
|
||||
fingerprints: input.fingerprints === undefined ? existingFm?.fingerprints : input.fingerprints,
|
||||
};
|
||||
|
||||
if (input.content) {
|
||||
finalContent = input.content;
|
||||
} else {
|
||||
const editResult = applySqlEdits(existing?.content ?? '', input.replacements ?? []);
|
||||
if (!editResult.success) {
|
||||
return {
|
||||
markdown: `Edit errors: ${editResult.errors.join('; ')}`,
|
||||
structured: { success: false, key: input.key },
|
||||
};
|
||||
}
|
||||
finalContent = editResult.sql;
|
||||
}
|
||||
|
||||
await wikiService.writePage(scope, scopeId, input.key, finalFm, finalContent, SYSTEM_AUTHOR, SYSTEM_EMAIL);
|
||||
if (!skipIndex) {
|
||||
await wikiService.syncSinglePage(scope, scopeId, input.key, finalFm, finalContent);
|
||||
}
|
||||
|
||||
await this.knowledgeRepository.createEvent({
|
||||
blockId: null,
|
||||
eventType: existing ? 'BLOCK_UPDATED' : 'BLOCK_CREATED',
|
||||
actorId: context.userId,
|
||||
chatId: null,
|
||||
messageId: null,
|
||||
payload: {
|
||||
pageKey: input.key,
|
||||
previousContent: existing ? existing.content.slice(0, 500) : null,
|
||||
},
|
||||
});
|
||||
|
||||
const action = existing ? 'updated' : 'created';
|
||||
if (context.session) {
|
||||
context.session.actions.push({ target: 'wiki', type: action, key: input.key, detail: input.summary });
|
||||
}
|
||||
|
||||
return {
|
||||
markdown: `Page "${input.key}" ${action}.`,
|
||||
structured: { success: true, key: input.key, action },
|
||||
};
|
||||
}
|
||||
}
|
||||
55
packages/context/src/wiki/types.ts
Normal file
55
packages/context/src/wiki/types.ts
Normal file
|
|
@ -0,0 +1,55 @@
|
|||
export type WikiScope = 'GLOBAL' | 'USER';
|
||||
|
||||
export interface HistoricSqlWikiUsageFrontmatter {
|
||||
executions: number;
|
||||
distinct_users: number;
|
||||
first_seen: string;
|
||||
last_seen: string;
|
||||
p50_runtime_ms: number | null;
|
||||
p95_runtime_ms: number | null;
|
||||
error_rate: number;
|
||||
rows_produced?: number;
|
||||
}
|
||||
|
||||
export interface WikiFrontmatter {
|
||||
summary: string;
|
||||
tags?: string[];
|
||||
refs?: string[];
|
||||
sl_refs?: string[];
|
||||
usage_mode: 'always' | 'auto' | 'never';
|
||||
sort_order?: number;
|
||||
source?: string;
|
||||
intent?: string;
|
||||
tables?: string[];
|
||||
representative_sql?: string;
|
||||
usage?: HistoricSqlWikiUsageFrontmatter;
|
||||
fingerprints?: string[];
|
||||
}
|
||||
|
||||
export interface WikiPage {
|
||||
pageKey: string;
|
||||
frontmatter: WikiFrontmatter;
|
||||
content: string;
|
||||
}
|
||||
|
||||
export interface WikiPageWithScope extends WikiPage {
|
||||
scope: WikiScope;
|
||||
}
|
||||
|
||||
export type WikiSearchMatchReason = 'lexical' | 'semantic' | 'token' | (string & {});
|
||||
|
||||
export interface WikiSearchLaneSummary {
|
||||
lane: string;
|
||||
status: 'available' | 'skipped' | 'failed';
|
||||
requestedCandidatePoolLimit: number;
|
||||
effectiveCandidatePoolLimit: number;
|
||||
returnedCandidateCount: number;
|
||||
weight: number;
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
export interface WikiSearchMetadata {
|
||||
score: number;
|
||||
matchReasons: WikiSearchMatchReason[];
|
||||
lanes?: WikiSearchLaneSummary[];
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue