mirror of
https://github.com/Kaelio/ktx.git
synced 2026-06-25 08:48:08 +02:00
chore(workspace): gate dead-code with knip production mode (#196)
* refactor(workspace): relocate @ktx/llm source into packages/cli/src/llm * refactor(workspace): rewrite @ktx/llm imports to relative paths * refactor(workspace): fold internal packages into cli * chore(workspace): gate dead-code with knip production mode Turn on production-mode knip plus an autofix run in pre-commit and the `pnpm dead-code` script, document the `/** @internal */` convention for test-only exports in AGENTS.md, annotate test-only exports across the CLI with that JSDoc, and drop dead exports/wrappers the new gate surfaced (e.g. `cli-project.ts`, `lookerRuntimeSourceToFileAdapterSource`, `createLocalScanEnrichmentProvidersFromConfig`, `PGLITE_OWNER_PROCESS_BACKEND_CAPABILITIES`, stale type re-exports). Replace the loose `ignoreIssues` allowlist in `knip.json` with explicit production entries so cross-package barrel leaks are caught. * refactor(cli): delete internal barrel index.ts files The 34 `index.ts` re-export barrels inside `packages/cli/src/` were holdovers from the pre-fold multi-workspace structure. Post-fold-in they served no production purpose: external consumers go through the single package main entry, and in-repo callers mostly imported through them only because the path was short. Internally, knip flagged most barrel re-exports as production-dead (only reached via tests). This change: - Deletes every internal barrel except `packages/cli/src/index.ts` (the published package entry). - Rewrites ~270 source/test files to import each name directly from the file that defines it. - Moves `tools/warehouse-verification/index.ts` to `create-warehouse-verification-tools.ts` (the function it defined locally) and updates its single consumer. - Renames `search/backend-conformance.ts` → `.test-utils.ts` to match the existing test-helper file convention. - Deletes 13 dead test-only chains (dbt-descriptions/*, live-database/extracted-schema, live-database/structural-sync, relationship-* feedback/review chain) plus their tests and a cascading orphan integration test. - Updates test mocks that pointed at deleted barrel paths (notion-client, connector barrels in scan/local-scan-connectors tests) to mock the source files instead. - Points the maintainer benchmark script (`scripts/relationship-benchmark-report.mjs`) at source files instead of `dist/context/scan/index.js`. - Drops the barrel `!` entries from `knip.json`; adds explicit production entries only for the benchmark code reached via dist by the maintainer script. Net: 413 files changed, ~1.2k insertions, ~9.4k deletions. `pnpm run dead-code` (Biome + knip default + knip production) and `pnpm run type-check` are clean; 2277 tests pass. * refactor(workspace): rename @ktx/cli to @kaelio/ktx and pack it directly Promote the CLI workspace package to the public name `@kaelio/ktx` and drop the separate `scripts/build-public-npm-package.mjs` wrapper. The CLI package is now publishable in place (`publishConfig.access: public`, `provenance: true`), so artifact packing uses `pnpm pack` against `packages/cli/` instead of assembling a parallel package tree. Updates all workspace filter invocations, docs, tests, and release readiness checks to reference the new package name, and folds the tarball-name helper into `scripts/public-npm-release-metadata.mjs`. * docs: align "agent clients" and "data agents" terminology Replace "client agents" with "agent clients" and "database agents" with "data agents" across AGENTS.md, README.md, the docs-site copy, and the matching setup-agents test description, matching the canonical vocabulary in docs/terminology.md. Also moves packages/cli/tsconfig.json's tsBuildInfoFile from node_modules/.cache/ to dist/.tsbuildinfo so incremental builds survive node_modules reinstalls. * refactor(release): single source of truth for package version Make packages/cli/package.json the single source of truth for the @kaelio/ktx version. publicNpmPackageVersion() now reads it directly, so artifact filenames, release-readiness checks, and the Python wheel version all derive from one field. The duplicate release-policy.json.publicNpmPackageVersion is removed. Previously the two fields could drift: tarballs were named kaelio-ktx-0.4.1.tgz while internally containing @kaelio/ktx@0.0.0-private. - update-public-release-version.mjs rewrites both Python pyproject.toml files (ktx-daemon, ktx-sl) alongside the npm package.jsons, normalizing the version for PEP 440 (e.g. 0.1.0-rc.2 -> 0.1.0rc2). - semantic-release-config.cjs adds the two pyproject.toml files to @semantic-release/git assets so the release commit back to main carries every version source in lockstep. - The six "?? '0.0.0-private'" fallback literals across the CLI are replaced with "?? getKtxCliPackageInfo().version", and createDefaultKtxMcpServer makes its version arg required. - docs/release.md describes the actual commit-back model: the dev tree always reflects the most recent release; no sentinel pin to maintain. Verified: pnpm run artifacts:build now produces kaelio-ktx-0.4.1.tgz and kaelio_ktx-0.4.1-py3-none-any.whl with @kaelio/ktx@0.4.1 inside. Full type-check, dead-code, and 2287 vitests + 173 script tests pass. * refactor(cli): inject embedding provider resolution and detect sentence-transformers runtime Make resolveProjectEmbeddingProvider and runtimeIo injectable in ingest and scan command entrypoints so tests can stub them, and teach resolvePublicIngestRuntimeRequirements to flag the local-embeddings runtime feature when ktx.yaml selects sentence-transformers. * chore(cli): mark buildLocalStatsStatus and LocalStatsStatus as @internal Both symbols are consumed only by status-project.test.ts. Annotating with /** @internal */ keeps knip's production-mode check clean without changing runtime behavior. * fix(cli): use real package metadata in print-command-tree The stubbed package name embedded a forbidden product identifier that tripped the boundary check in CI. Read the metadata from package.json instead — keeps the rendered tree unchanged and removes a duplicate source of truth. * feat(cli): show embedding coverage in `ktx status`, drop duplicate disk counts Inline `(N embedded)` next to the Wiki scope counts and Semantic-layer source counts, computed with `SUM(embedding_json IS NOT NULL)` over `knowledge_pages` and `local_sl_sources`. Rename the "Knowledge" label to "Wiki" (canonical per `docs/terminology.md`) and rename the matching `localStats.knowledgePages` field to `localStats.wikiPages`. Drop `wiki=N md` and `semantic-layer=N yaml` from the Disk row — those duplicated the per-surface rows above. Disk now reports only actual byte usage (db, cache, raw-sources). The unused `wikiGlobalMarkdownCount` / `semanticLayerYamlCount` fields, the `isMarkdownEntry` / `isYamlEntry` helpers, and the `filter` arg on `summarizeDir` are removed.
This commit is contained in:
parent
a1cfb03d73
commit
2366b00301
1002 changed files with 2286 additions and 12051 deletions
504
packages/cli/src/context/memory/local-memory.ts
Normal file
504
packages/cli/src/context/memory/local-memory.ts
Normal file
|
|
@ -0,0 +1,504 @@
|
|||
import { join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import YAML from 'yaml';
|
||||
import { localConnectionInfoFromConfig } from '../../context/connections/local-warehouse-descriptor.js';
|
||||
import type { KtxEmbeddingPort } from '../../context/core/embedding.js';
|
||||
import type { KtxFileStorePort, KtxFileWriteResult } from '../../context/core/file-store.js';
|
||||
import { type KtxLogger, noopLogger } from '../../context/core/config.js';
|
||||
import { SessionWorktreeService } from '../../context/core/session-worktree.service.js';
|
||||
import type { KtxSemanticLayerComputePort } from '../../context/daemon/semantic-layer-compute.js';
|
||||
import { KtxIngestEmbeddingPortAdapter } from '../../context/llm/embedding-port.js';
|
||||
import { createLocalKtxLlmRuntimeFromConfig } from '../../context/llm/local-config.js';
|
||||
import { RuntimeAgentRunner, type AgentRunnerPort, type KtxLlmRuntimePort, type KtxRuntimeToolSet } from '../../context/llm/runtime-port.js';
|
||||
import type { KtxEmbeddingProvider } from '../../llm/types.js';
|
||||
import type { KtxLocalProject } from '../../context/project/project.js';
|
||||
import { PromptService } from '../../context/prompts/prompt.service.js';
|
||||
import { SkillsRegistryService } from '../../context/skills/skills-registry.service.js';
|
||||
import type { KtxConnectionInfo, KtxQueryResult, SlConnectionCatalogPort, SlPythonPort, SlSourcesIndexPort } from '../../context/sl/ports.js';
|
||||
import { SemanticLayerService } from '../../context/sl/semantic-layer.service.js';
|
||||
import type { SemanticLayerSource } from '../../context/sl/types.js';
|
||||
import { SlDiscoverTool } from '../../context/sl/tools/sl-discover.tool.js';
|
||||
import { SlEditSourceTool } from '../../context/sl/tools/sl-edit-source.tool.js';
|
||||
import { SlReadSourceTool } from '../../context/sl/tools/sl-read-source.tool.js';
|
||||
import { SlRollbackTool } from '../../context/sl/tools/sl-rollback.tool.js';
|
||||
import { SlSearchService } from '../../context/sl/sl-search.service.js';
|
||||
import { SlValidateTool } from '../../context/sl/tools/sl-validate.tool.js';
|
||||
import type { SlValidationDeps } from '../../context/sl/tools/sl-warehouse-validation.js';
|
||||
import type { SlValidatorPort } from '../../context/sl/sl-validator.port.js';
|
||||
import { SlWriteSourceTool } from '../../context/sl/tools/sl-write-source.tool.js';
|
||||
import { SqliteSlSourcesIndex } from '../../context/sl/sqlite-sl-sources-index.js';
|
||||
import { sourceDefinitionSchema, sourceOverlaySchema } from '../../context/sl/schemas.js';
|
||||
import { BaseTool, type ToolContext } from '../../context/tools/base-tool.js';
|
||||
import type { GitAuthorResolverPort } from '../../context/tools/authors.js';
|
||||
import type { KnowledgeEventPort, KnowledgeIndexPort, KnowledgeIndexPageListing } from '../../context/wiki/ports.js';
|
||||
import { KnowledgeWikiService } from '../../context/wiki/knowledge-wiki.service.js';
|
||||
import { searchLocalKnowledgePages } from '../../context/wiki/local-knowledge.js';
|
||||
import { WikiListTagsTool } from '../../context/wiki/tools/wiki-list-tags.tool.js';
|
||||
import { WikiReadTool } from '../../context/wiki/tools/wiki-read.tool.js';
|
||||
import { WikiRemoveTool } from '../../context/wiki/tools/wiki-remove.tool.js';
|
||||
import { WikiSearchTool } from '../../context/wiki/tools/wiki-search.tool.js';
|
||||
import { WikiWriteTool } from '../../context/wiki/tools/wiki-write.tool.js';
|
||||
import { LocalMemoryRunStore } from './local-memory-runs.js';
|
||||
import { MemoryAgentService } from './memory-agent.service.js';
|
||||
import { MemoryIngestService } from './memory-runs.js';
|
||||
import type {
|
||||
MemoryConnectionPort,
|
||||
MemoryFileStorePort,
|
||||
MemoryKnowledgeSlRefsPort,
|
||||
MemorySlSourceReconcilerPort,
|
||||
MemoryToolSetLike,
|
||||
MemoryToolsetFactoryPort,
|
||||
} from './types.js';
|
||||
|
||||
const promptsDir = fileURLToPath(new URL('../../prompts', import.meta.url));
|
||||
const skillsDir = fileURLToPath(new URL('../../skills', import.meta.url));
|
||||
const LOCAL_AUTHOR = { name: 'KTX Local', email: 'local@ktx.local' };
|
||||
const LOCAL_SHAPE_WARNING = 'Local memory ingest validates semantic-layer YAML shape only.';
|
||||
|
||||
export interface CreateLocalProjectMemoryIngestOptions {
|
||||
llmRuntime?: KtxLlmRuntimePort;
|
||||
agentRunner?: AgentRunnerPort;
|
||||
memoryModel?: string;
|
||||
semanticLayerCompute?: KtxSemanticLayerComputePort;
|
||||
queryExecutor?: { execute(input: { connectionId: string; sql: string; maxRows?: number }): Promise<KtxQueryResult> };
|
||||
runIdFactory?: () => string;
|
||||
logger?: KtxLogger;
|
||||
embeddingProvider?: KtxEmbeddingProvider | null;
|
||||
}
|
||||
|
||||
export function createLocalProjectMemoryIngest(
|
||||
project: KtxLocalProject,
|
||||
options: CreateLocalProjectMemoryIngestOptions = {},
|
||||
): MemoryIngestService {
|
||||
const logger = options.logger ?? noopLogger;
|
||||
const rootFileStore = new LocalMemoryFileStore(project.fileStore);
|
||||
const embedding = options.embeddingProvider
|
||||
? new KtxIngestEmbeddingPortAdapter(options.embeddingProvider)
|
||||
: new NoopEmbeddingPort();
|
||||
if (!options.embeddingProvider && project.config.ingest.embeddings.backend !== 'none') {
|
||||
// Memory-agent search (SlSearch, wiki) embeds against this port. With Noop the
|
||||
// configured backend is silently inert — the agent will see empty vectors and
|
||||
// rank results against zeros. Surface that so the caller knows to plumb the
|
||||
// resolved embedding provider through.
|
||||
logger.warn(
|
||||
`[memory-ingest] embeddings backend "${project.config.ingest.embeddings.backend}" is configured but no embedding provider was passed; semantic search will fall back to a no-op embedding port.`,
|
||||
);
|
||||
}
|
||||
const knowledgeIndex = new LocalKnowledgeIndex(project);
|
||||
const knowledgeEvents = new NoopKnowledgeEventPort();
|
||||
const knowledgeSlRefs = new NoopKnowledgeSlRefsPort();
|
||||
const connections = new LocalMemoryConnections(project, options.queryExecutor);
|
||||
const slPython = new LocalSlPythonPort(options.semanticLayerCompute);
|
||||
const semanticLayerService = new SemanticLayerService(rootFileStore, connections, slPython, logger);
|
||||
const slSourcesRepository = new SqliteSlSourcesIndex({ dbPath: join(project.projectDir, '.ktx', 'db.sqlite') });
|
||||
const slSearchService = new SlSearchService(embedding, slSourcesRepository, logger);
|
||||
const wikiService = new KnowledgeWikiService(rootFileStore, embedding, knowledgeIndex, project.git, logger);
|
||||
const authorResolver = new LocalAuthorResolver();
|
||||
const llmRuntime =
|
||||
options.llmRuntime ?? createLocalKtxLlmRuntimeFromConfig(project.config.llm, { projectDir: project.projectDir });
|
||||
const toolsetFactory = new LocalMemoryToolsetFactory({
|
||||
project,
|
||||
embedding,
|
||||
wikiService,
|
||||
knowledgeIndex,
|
||||
knowledgeEvents,
|
||||
semanticLayerService,
|
||||
slSearchService,
|
||||
authorResolver,
|
||||
slSourcesRepository,
|
||||
connections,
|
||||
});
|
||||
const agentRunner =
|
||||
options.agentRunner ??
|
||||
new RuntimeAgentRunner(requireLlmRuntime(llmRuntime));
|
||||
const memoryAgent = new MemoryAgentService({
|
||||
settings: {
|
||||
knowledge: { userScopedKnowledgeEnabled: false },
|
||||
slValidation: { probeRowCount: 0 },
|
||||
llm: { memoryIngestionModel: project.config.llm.models.default ?? 'local-memory-model' },
|
||||
},
|
||||
promptService: new PromptService({ promptsDir, partials: [] }),
|
||||
skillsRegistry: new SkillsRegistryService({ skillsDir }),
|
||||
wikiService,
|
||||
knowledgeIndex,
|
||||
knowledgeSlRefs,
|
||||
semanticLayerService,
|
||||
slSearchService,
|
||||
connections,
|
||||
rootFileStore,
|
||||
gitService: project.git,
|
||||
lockingService: new LocalMemoryLock(),
|
||||
slSourcesRepository,
|
||||
sessionWorktreeService: new SessionWorktreeService({
|
||||
coreConfig: project.coreConfig,
|
||||
gitService: project.git,
|
||||
configService: rootFileStore,
|
||||
}),
|
||||
semanticLayerSourceReconciler: new NoopSemanticLayerSourceReconciler(),
|
||||
agentRunner,
|
||||
slValidator: new LocalShapeOnlySlValidator(),
|
||||
toolsetFactory,
|
||||
logger,
|
||||
});
|
||||
return new MemoryIngestService({
|
||||
memoryAgent,
|
||||
runs: new LocalMemoryRunStore({ projectDir: project.projectDir, idFactory: options.runIdFactory }),
|
||||
});
|
||||
}
|
||||
|
||||
function requireLlmRuntime(runtime: KtxLlmRuntimePort | null | undefined): KtxLlmRuntimePort {
|
||||
if (!runtime) {
|
||||
throw new Error('createLocalProjectMemoryIngest requires llm.provider.backend or an injected agentRunner');
|
||||
}
|
||||
return runtime;
|
||||
}
|
||||
|
||||
class LocalMemoryFileStore implements MemoryFileStorePort {
|
||||
constructor(private readonly fileStore: MemoryFileStorePort | KtxFileStorePort) {}
|
||||
|
||||
forWorktree(workdir: string): LocalMemoryFileStore {
|
||||
return new LocalMemoryFileStore(this.fileStore.forWorktree(workdir) as KtxFileStorePort);
|
||||
}
|
||||
|
||||
writeFile(...args: Parameters<KtxFileStorePort['writeFile']>): Promise<KtxFileWriteResult> {
|
||||
return this.fileStore.writeFile(...args);
|
||||
}
|
||||
|
||||
readFile(...args: Parameters<KtxFileStorePort['readFile']>) {
|
||||
return this.fileStore.readFile(...args);
|
||||
}
|
||||
|
||||
deleteFile(...args: Parameters<KtxFileStorePort['deleteFile']>) {
|
||||
return this.fileStore.deleteFile(...args);
|
||||
}
|
||||
|
||||
listFiles(...args: Parameters<KtxFileStorePort['listFiles']>) {
|
||||
return this.fileStore.listFiles(...args);
|
||||
}
|
||||
|
||||
getFileHistory(...args: Parameters<KtxFileStorePort['getFileHistory']>) {
|
||||
return this.fileStore.getFileHistory(...args);
|
||||
}
|
||||
|
||||
async enqueueCommitMessageJobForExternalCommit(): Promise<void> {}
|
||||
}
|
||||
|
||||
class NoopEmbeddingPort implements KtxEmbeddingPort {
|
||||
readonly maxBatchSize = 64;
|
||||
|
||||
async computeEmbedding(): Promise<number[]> {
|
||||
return [];
|
||||
}
|
||||
|
||||
async computeEmbeddingsBulk(texts: string[]): Promise<number[][]> {
|
||||
return texts.map(() => []);
|
||||
}
|
||||
}
|
||||
|
||||
class LocalKnowledgeIndex implements KnowledgeIndexPort {
|
||||
constructor(private readonly project: KtxLocalProject) {}
|
||||
|
||||
async upsertPage(): Promise<void> {}
|
||||
|
||||
async applyDiffTransactional(): Promise<void> {}
|
||||
|
||||
async getExistingSearchTexts(): Promise<Map<string, { searchText: string; hasEmbedding: boolean }>> {
|
||||
return new Map();
|
||||
}
|
||||
|
||||
async deleteStale(): Promise<number> {
|
||||
return 0;
|
||||
}
|
||||
|
||||
async deleteByScope(): Promise<number> {
|
||||
return 0;
|
||||
}
|
||||
|
||||
async deleteByKey(): Promise<number> {
|
||||
return 0;
|
||||
}
|
||||
|
||||
async findPageByKey(scope: string, scopeId: string | null, pageKey: string) {
|
||||
const path = this.pagePath(scope, scopeId, pageKey);
|
||||
try {
|
||||
await this.project.fileStore.readFile(path);
|
||||
return { page_key: pageKey };
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async listPagesForUser(userId: string) {
|
||||
const pages: KnowledgeIndexPageListing[] = [];
|
||||
for (const scope of [
|
||||
{ scope: 'GLOBAL', scopeId: null, dir: 'wiki/global' },
|
||||
{ scope: 'USER', scopeId: userId, dir: `wiki/user/${userId}` },
|
||||
]) {
|
||||
const listed = await this.project.fileStore.listFiles(scope.dir, true);
|
||||
for (const file of listed.files.filter((entry) => entry.endsWith('.md'))) {
|
||||
const pageKey = file.replace(/\.md$/, '');
|
||||
const raw = await this.project.fileStore.readFile(`${scope.dir}/${file}`);
|
||||
const parsed = parseWiki(raw.content);
|
||||
pages.push({
|
||||
page_key: pageKey,
|
||||
summary: parsed.summary,
|
||||
scope: scope.scope,
|
||||
scope_id: scope.scopeId,
|
||||
tags: parseWikiTags(raw.content),
|
||||
});
|
||||
}
|
||||
}
|
||||
return pages.sort((a, b) => a.page_key.localeCompare(b.page_key));
|
||||
}
|
||||
|
||||
async getUserPageCount(userId: string): Promise<number> {
|
||||
return (await this.listPagesForUser(userId)).filter((page) => page.scope === 'USER').length;
|
||||
}
|
||||
|
||||
async incrementUsageCount(): Promise<void> {}
|
||||
|
||||
async searchRRF(_userId: string, _embedding: number[] | null, queryText: string, limit: number) {
|
||||
const pages = await this.listPagesForUser(_userId);
|
||||
return pages
|
||||
.map((page) => ({
|
||||
pageKey: page.page_key,
|
||||
summary: page.summary,
|
||||
rrfScore: scoreText(`${page.page_key} ${page.summary}`, queryText),
|
||||
}))
|
||||
.filter((page) => page.rrfScore > 0)
|
||||
.sort((a, b) => b.rrfScore - a.rrfScore || a.pageKey.localeCompare(b.pageKey))
|
||||
.slice(0, limit);
|
||||
}
|
||||
|
||||
private pagePath(scope: string, scopeId: string | null, pageKey: string): string {
|
||||
return scope === 'GLOBAL' ? `wiki/global/${pageKey}.md` : `wiki/user/${scopeId}/${pageKey}.md`;
|
||||
}
|
||||
}
|
||||
|
||||
class NoopKnowledgeEventPort implements KnowledgeEventPort {
|
||||
async createEvent(): Promise<void> {}
|
||||
}
|
||||
|
||||
class NoopKnowledgeSlRefsPort implements MemoryKnowledgeSlRefsPort {
|
||||
async syncFromWiki(): Promise<{ inserted: number; deleted: number }> {
|
||||
return { inserted: 0, deleted: 0 };
|
||||
}
|
||||
}
|
||||
|
||||
class LocalMemoryConnections implements MemoryConnectionPort, SlConnectionCatalogPort {
|
||||
constructor(
|
||||
private readonly project: KtxLocalProject,
|
||||
private readonly queryExecutor?: {
|
||||
execute(input: { connectionId: string; sql: string; maxRows?: number }): Promise<KtxQueryResult>;
|
||||
},
|
||||
) {}
|
||||
|
||||
async listEnabledConnections(ids: string[]): Promise<KtxConnectionInfo[]> {
|
||||
return ids
|
||||
.map((id) => localConnectionInfoFromConfig(id, this.project.config.connections[id]))
|
||||
.filter((connection): connection is KtxConnectionInfo => connection !== null);
|
||||
}
|
||||
|
||||
async getConnectionById(connectionId: string): Promise<KtxConnectionInfo> {
|
||||
const connection = localConnectionInfoFromConfig(connectionId, this.project.config.connections[connectionId]);
|
||||
if (!connection) {
|
||||
throw new Error(`Connection not found: ${connectionId}`);
|
||||
}
|
||||
return connection;
|
||||
}
|
||||
|
||||
async executeQuery(connectionId: string, sql: string): Promise<KtxQueryResult> {
|
||||
if (!this.queryExecutor) {
|
||||
throw new Error('Local memory capture has no query executor configured');
|
||||
}
|
||||
return this.queryExecutor.execute({ connectionId, sql });
|
||||
}
|
||||
}
|
||||
|
||||
class LocalSlPythonPort implements SlPythonPort {
|
||||
constructor(private readonly compute?: KtxSemanticLayerComputePort) {}
|
||||
|
||||
async validateSources(input: Parameters<SlPythonPort['validateSources']>[0]) {
|
||||
if (!this.compute) {
|
||||
return {
|
||||
data: {
|
||||
errors: [],
|
||||
warnings: [LOCAL_SHAPE_WARNING],
|
||||
per_source_warnings: {},
|
||||
},
|
||||
};
|
||||
}
|
||||
const result = await this.compute.validateSources({
|
||||
sources: input.sources,
|
||||
dialect: input.dialect,
|
||||
recentlyTouched: input.recently_touched,
|
||||
});
|
||||
return {
|
||||
data: {
|
||||
errors: result.errors,
|
||||
warnings: result.warnings,
|
||||
per_source_warnings: result.perSourceWarnings,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async query(input: Parameters<SlPythonPort['query']>[0]) {
|
||||
if (!this.compute) {
|
||||
return { error: 'Local memory capture has no semantic compute adapter configured' };
|
||||
}
|
||||
const result = await this.compute.query({
|
||||
sources: input.sources,
|
||||
dialect: input.dialect,
|
||||
query: input.query,
|
||||
});
|
||||
return { data: { sql: result.sql, plan: result.plan } };
|
||||
}
|
||||
}
|
||||
|
||||
class LocalAuthorResolver implements GitAuthorResolverPort {
|
||||
async resolve() {
|
||||
return LOCAL_AUTHOR;
|
||||
}
|
||||
}
|
||||
|
||||
class LocalMemoryLock {
|
||||
async withLock<T>(_key: 'config:repo', fn: () => Promise<T>): Promise<T> {
|
||||
return fn();
|
||||
}
|
||||
}
|
||||
|
||||
class NoopSemanticLayerSourceReconciler implements MemorySlSourceReconcilerPort {
|
||||
async upsertRow(): Promise<void> {}
|
||||
}
|
||||
|
||||
class LocalShapeOnlySlValidator implements SlValidatorPort<SlValidationDeps> {
|
||||
async validateSingleSource(deps: SlValidationDeps, connectionId: string, sourceName: string) {
|
||||
try {
|
||||
const file = await deps.semanticLayerService.readSourceFile(connectionId, sourceName);
|
||||
const parsed = YAML.parse(file.content) as SemanticLayerSource;
|
||||
const isOverlay = parsed.table == null && parsed.sql == null;
|
||||
const result = (isOverlay ? sourceOverlaySchema : sourceDefinitionSchema).safeParse(parsed);
|
||||
return result.success
|
||||
? { errors: [], warnings: [LOCAL_SHAPE_WARNING] }
|
||||
: {
|
||||
errors: result.error.issues.map(
|
||||
(issue) => `${sourceName}: ${issue.path.join('.') || 'source'} ${issue.message}`,
|
||||
),
|
||||
warnings: [],
|
||||
};
|
||||
} catch (error) {
|
||||
return { errors: [`${sourceName}: ${error instanceof Error ? error.message : String(error)}`], warnings: [] };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
class LocalMemoryToolSet implements MemoryToolSetLike {
|
||||
constructor(private readonly tools: BaseTool[]) {}
|
||||
|
||||
toRuntimeTools(context: ToolContext): KtxRuntimeToolSet {
|
||||
return Object.fromEntries(this.tools.map((tool) => [tool.name, tool.toRuntimeTool(context)]));
|
||||
}
|
||||
}
|
||||
|
||||
class LocalMemoryToolsetFactory implements MemoryToolsetFactoryPort {
|
||||
private readonly wikiTools: BaseTool[];
|
||||
private readonly slTools: BaseTool[];
|
||||
|
||||
constructor(deps: {
|
||||
project: KtxLocalProject;
|
||||
embedding: KtxEmbeddingPort;
|
||||
wikiService: KnowledgeWikiService;
|
||||
knowledgeIndex: KnowledgeIndexPort;
|
||||
knowledgeEvents: KnowledgeEventPort;
|
||||
semanticLayerService: SemanticLayerService;
|
||||
slSearchService: SlSearchService;
|
||||
authorResolver: GitAuthorResolverPort;
|
||||
slSourcesRepository: SlSourcesIndexPort;
|
||||
connections: SlConnectionCatalogPort;
|
||||
}) {
|
||||
const slDeps = {
|
||||
semanticLayerService: deps.semanticLayerService,
|
||||
slSearchService: deps.slSearchService,
|
||||
authorResolver: deps.authorResolver,
|
||||
};
|
||||
this.wikiTools = [
|
||||
new WikiReadTool(deps.wikiService, deps.knowledgeIndex),
|
||||
new WikiSearchTool({
|
||||
search: async (input) => {
|
||||
const results = await searchLocalKnowledgePages(deps.project, {
|
||||
userId: input.userId,
|
||||
query: input.query,
|
||||
limit: input.limit,
|
||||
embeddingService: deps.embedding,
|
||||
});
|
||||
return {
|
||||
results: results.slice(0, input.limit).map((result) => ({
|
||||
key: result.key,
|
||||
path: result.path,
|
||||
summary: result.summary,
|
||||
score: result.score,
|
||||
matchReasons: result.matchReasons,
|
||||
lanes: result.lanes,
|
||||
})),
|
||||
totalFound: results.length,
|
||||
};
|
||||
},
|
||||
}),
|
||||
new WikiListTagsTool(deps.knowledgeIndex),
|
||||
new WikiWriteTool(deps.wikiService, deps.knowledgeIndex, deps.knowledgeEvents),
|
||||
new WikiRemoveTool(deps.wikiService, deps.knowledgeIndex, deps.knowledgeEvents),
|
||||
];
|
||||
this.slTools = [
|
||||
new SlDiscoverTool(slDeps, { maxSources: 25, minRrfScore: 0, maxDetailedSources: 5 }),
|
||||
new SlEditSourceTool(slDeps),
|
||||
new SlReadSourceTool(slDeps),
|
||||
new SlWriteSourceTool(slDeps),
|
||||
new SlValidateTool(slDeps),
|
||||
new SlRollbackTool(deps.slSourcesRepository, deps.connections, 0),
|
||||
];
|
||||
}
|
||||
|
||||
createIngestWuToolset(): MemoryToolSetLike {
|
||||
return new LocalMemoryToolSet([...this.wikiTools, ...this.slTools]);
|
||||
}
|
||||
|
||||
createToolset(): MemoryToolSetLike {
|
||||
return new LocalMemoryToolSet(this.wikiTools);
|
||||
}
|
||||
}
|
||||
|
||||
function parseWiki(raw: string): { summary: string; content: string } {
|
||||
const match = raw.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
|
||||
if (!match) {
|
||||
return { summary: '', content: raw.trim() };
|
||||
}
|
||||
const frontmatter = (YAML.parse(match[1]) ?? {}) as Record<string, unknown>;
|
||||
return {
|
||||
summary: typeof frontmatter.summary === 'string' ? frontmatter.summary : '',
|
||||
content: match[2].trim(),
|
||||
};
|
||||
}
|
||||
|
||||
function parseWikiTags(raw: string): string[] {
|
||||
const match = raw.match(/^---\n([\s\S]*?)\n---\n?/);
|
||||
if (!match) {
|
||||
return [];
|
||||
}
|
||||
const frontmatter = (YAML.parse(match[1]) ?? {}) as Record<string, unknown>;
|
||||
return Array.isArray(frontmatter.tags)
|
||||
? frontmatter.tags.filter((tag): tag is string => typeof tag === 'string')
|
||||
: [];
|
||||
}
|
||||
|
||||
function scoreText(text: string, query: string): number {
|
||||
const normalized = query.toLowerCase().trim();
|
||||
if (!normalized) {
|
||||
return 0;
|
||||
}
|
||||
const haystack = text.toLowerCase();
|
||||
if (haystack.includes(normalized)) {
|
||||
return 1;
|
||||
}
|
||||
const words = normalized.split(/\s+/).filter(Boolean);
|
||||
return words.filter((word) => haystack.includes(word)).length / Math.max(words.length, 1);
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue