import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; import YAML from 'yaml'; import { localConnectionInfoFromConfig } from '../../context/connections/local-warehouse-descriptor.js'; import type { KtxEmbeddingPort } from '../../context/core/embedding.js'; import type { KtxFileStorePort, KtxFileWriteResult } from '../../context/core/file-store.js'; import { type KtxLogger, noopLogger } from '../../context/core/config.js'; import { SessionWorktreeService } from '../../context/core/session-worktree.service.js'; import type { KtxSemanticLayerComputePort } from '../../context/daemon/semantic-layer-compute.js'; import { KtxIngestEmbeddingPortAdapter } from '../../context/llm/embedding-port.js'; import { createLocalKtxLlmRuntimeFromConfig } from '../../context/llm/local-config.js'; import { RuntimeAgentRunner, type AgentRunnerPort, type KtxLlmRuntimePort, type KtxRuntimeToolSet } from '../../context/llm/runtime-port.js'; import type { KtxEmbeddingProvider } from '../../llm/types.js'; import type { KtxLocalProject } from '../../context/project/project.js'; import { PromptService } from '../../context/prompts/prompt.service.js'; import { SkillsRegistryService } from '../../context/skills/skills-registry.service.js'; import type { KtxConnectionInfo, KtxQueryResult, SlConnectionCatalogPort, SlPythonPort, SlSourcesIndexPort } from '../../context/sl/ports.js'; import { SemanticLayerService } from '../../context/sl/semantic-layer.service.js'; import type { SemanticLayerSource } from '../../context/sl/types.js'; import { SlDiscoverTool } from '../../context/sl/tools/sl-discover.tool.js'; import { SlEditSourceTool } from '../../context/sl/tools/sl-edit-source.tool.js'; import { SlReadSourceTool } from '../../context/sl/tools/sl-read-source.tool.js'; import { SlRollbackTool } from '../../context/sl/tools/sl-rollback.tool.js'; import { SlSearchService } from '../../context/sl/sl-search.service.js'; import { SlValidateTool } from '../../context/sl/tools/sl-validate.tool.js'; import type { SlValidationDeps } from '../../context/sl/tools/sl-warehouse-validation.js'; import type { SlValidatorPort } from '../../context/sl/sl-validator.port.js'; import { SlWriteSourceTool } from '../../context/sl/tools/sl-write-source.tool.js'; import { SqliteSlSourcesIndex } from '../../context/sl/sqlite-sl-sources-index.js'; import { sourceDefinitionSchema, sourceOverlaySchema } from '../../context/sl/schemas.js'; import { BaseTool, type ToolContext } from '../../context/tools/base-tool.js'; import type { GitAuthorResolverPort } from '../../context/tools/authors.js'; import type { KnowledgeEventPort, KnowledgeIndexPort, KnowledgeIndexPageListing } from '../../context/wiki/ports.js'; import { KnowledgeWikiService } from '../../context/wiki/knowledge-wiki.service.js'; import { searchLocalKnowledgePages } from '../../context/wiki/local-knowledge.js'; import { WikiListTagsTool } from '../../context/wiki/tools/wiki-list-tags.tool.js'; import { WikiReadTool } from '../../context/wiki/tools/wiki-read.tool.js'; import { WikiRemoveTool } from '../../context/wiki/tools/wiki-remove.tool.js'; import { WikiSearchTool } from '../../context/wiki/tools/wiki-search.tool.js'; import { WikiWriteTool } from '../../context/wiki/tools/wiki-write.tool.js'; import { LocalMemoryRunStore } from './local-memory-runs.js'; import { MemoryAgentService } from './memory-agent.service.js'; import { MemoryIngestService } from './memory-runs.js'; import type { MemoryConnectionPort, MemoryFileStorePort, MemoryKnowledgeSlRefsPort, MemorySlSourceReconcilerPort, MemoryToolSetLike, MemoryToolsetFactoryPort, } from './types.js'; const promptsDir = fileURLToPath(new URL('../../prompts', import.meta.url)); const skillsDir = fileURLToPath(new URL('../../skills', import.meta.url)); const LOCAL_AUTHOR = { name: 'KTX Local', email: 'local@ktx.local' }; const LOCAL_SHAPE_WARNING = 'Local memory ingest validates semantic-layer YAML shape only.'; export interface CreateLocalProjectMemoryIngestOptions { llmRuntime?: KtxLlmRuntimePort; agentRunner?: AgentRunnerPort; memoryModel?: string; semanticLayerCompute?: KtxSemanticLayerComputePort; queryExecutor?: { execute(input: { connectionId: string; sql: string; maxRows?: number }): Promise }; runIdFactory?: () => string; logger?: KtxLogger; embeddingProvider?: KtxEmbeddingProvider | null; } export function createLocalProjectMemoryIngest( project: KtxLocalProject, options: CreateLocalProjectMemoryIngestOptions = {}, ): MemoryIngestService { const logger = options.logger ?? noopLogger; const rootFileStore = new LocalMemoryFileStore(project.fileStore); const embedding = options.embeddingProvider ? new KtxIngestEmbeddingPortAdapter(options.embeddingProvider) : new NoopEmbeddingPort(); if (!options.embeddingProvider && project.config.ingest.embeddings.backend !== 'none') { // Memory-agent search (SlSearch, wiki) embeds against this port. With Noop the // configured backend is silently inert — the agent will see empty vectors and // rank results against zeros. Surface that so the caller knows to plumb the // resolved embedding provider through. logger.warn( `[memory-ingest] embeddings backend "${project.config.ingest.embeddings.backend}" is configured but no embedding provider was passed; semantic search will fall back to a no-op embedding port.`, ); } const knowledgeIndex = new LocalKnowledgeIndex(project); const knowledgeEvents = new NoopKnowledgeEventPort(); const knowledgeSlRefs = new NoopKnowledgeSlRefsPort(); const connections = new LocalMemoryConnections(project, options.queryExecutor); const slPython = new LocalSlPythonPort(options.semanticLayerCompute); const semanticLayerService = new SemanticLayerService(rootFileStore, connections, slPython, logger); const slSourcesRepository = new SqliteSlSourcesIndex({ dbPath: join(project.projectDir, '.ktx', 'db.sqlite') }); const slSearchService = new SlSearchService(embedding, slSourcesRepository, logger); const wikiService = new KnowledgeWikiService(rootFileStore, embedding, knowledgeIndex, project.git, logger); const authorResolver = new LocalAuthorResolver(); const llmRuntime = options.llmRuntime ?? createLocalKtxLlmRuntimeFromConfig(project.config.llm, { projectDir: project.projectDir }); const toolsetFactory = new LocalMemoryToolsetFactory({ project, embedding, wikiService, knowledgeIndex, knowledgeEvents, semanticLayerService, slSearchService, authorResolver, slSourcesRepository, connections, }); const agentRunner = options.agentRunner ?? new RuntimeAgentRunner(requireLlmRuntime(llmRuntime)); const memoryAgent = new MemoryAgentService({ settings: { knowledge: { userScopedKnowledgeEnabled: false }, slValidation: { probeRowCount: 0 }, llm: { memoryIngestionModel: project.config.llm.models.default ?? 'local-memory-model' }, }, promptService: new PromptService({ promptsDir, partials: [] }), skillsRegistry: new SkillsRegistryService({ skillsDir }), wikiService, knowledgeIndex, knowledgeSlRefs, semanticLayerService, slSearchService, connections, rootFileStore, gitService: project.git, lockingService: new LocalMemoryLock(), slSourcesRepository, sessionWorktreeService: new SessionWorktreeService({ coreConfig: project.coreConfig, gitService: project.git, configService: rootFileStore, }), semanticLayerSourceReconciler: new NoopSemanticLayerSourceReconciler(), agentRunner, slValidator: new LocalShapeOnlySlValidator(), toolsetFactory, logger, }); return new MemoryIngestService({ memoryAgent, runs: new LocalMemoryRunStore({ projectDir: project.projectDir, idFactory: options.runIdFactory }), }); } function requireLlmRuntime(runtime: KtxLlmRuntimePort | null | undefined): KtxLlmRuntimePort { if (!runtime) { throw new Error('createLocalProjectMemoryIngest requires llm.provider.backend or an injected agentRunner'); } return runtime; } class LocalMemoryFileStore implements MemoryFileStorePort { constructor(private readonly fileStore: MemoryFileStorePort | KtxFileStorePort) {} forWorktree(workdir: string): LocalMemoryFileStore { return new LocalMemoryFileStore(this.fileStore.forWorktree(workdir) as KtxFileStorePort); } writeFile(...args: Parameters): Promise { return this.fileStore.writeFile(...args); } readFile(...args: Parameters) { return this.fileStore.readFile(...args); } deleteFile(...args: Parameters) { return this.fileStore.deleteFile(...args); } listFiles(...args: Parameters) { return this.fileStore.listFiles(...args); } getFileHistory(...args: Parameters) { return this.fileStore.getFileHistory(...args); } async enqueueCommitMessageJobForExternalCommit(): Promise {} } class NoopEmbeddingPort implements KtxEmbeddingPort { readonly maxBatchSize = 64; async computeEmbedding(): Promise { return []; } async computeEmbeddingsBulk(texts: string[]): Promise { return texts.map(() => []); } } class LocalKnowledgeIndex implements KnowledgeIndexPort { constructor(private readonly project: KtxLocalProject) {} async upsertPage(): Promise {} async applyDiffTransactional(): Promise {} async getExistingSearchTexts(): Promise> { return new Map(); } async deleteStale(): Promise { return 0; } async deleteByScope(): Promise { return 0; } async deleteByKey(): Promise { return 0; } async findPageByKey(scope: string, scopeId: string | null, pageKey: string) { const path = this.pagePath(scope, scopeId, pageKey); try { await this.project.fileStore.readFile(path); return { page_key: pageKey }; } catch { return null; } } async listPagesForUser(userId: string) { const pages: KnowledgeIndexPageListing[] = []; for (const scope of [ { scope: 'GLOBAL', scopeId: null, dir: 'wiki/global' }, { scope: 'USER', scopeId: userId, dir: `wiki/user/${userId}` }, ]) { const listed = await this.project.fileStore.listFiles(scope.dir, true); for (const file of listed.files.filter((entry) => entry.endsWith('.md'))) { const pageKey = file.replace(/\.md$/, ''); const raw = await this.project.fileStore.readFile(`${scope.dir}/${file}`); const parsed = parseWiki(raw.content); pages.push({ page_key: pageKey, summary: parsed.summary, scope: scope.scope, scope_id: scope.scopeId, tags: parseWikiTags(raw.content), }); } } return pages.sort((a, b) => a.page_key.localeCompare(b.page_key)); } async getUserPageCount(userId: string): Promise { return (await this.listPagesForUser(userId)).filter((page) => page.scope === 'USER').length; } async incrementUsageCount(): Promise {} async searchRRF(_userId: string, _embedding: number[] | null, queryText: string, limit: number) { const pages = await this.listPagesForUser(_userId); return pages .map((page) => ({ pageKey: page.page_key, summary: page.summary, rrfScore: scoreText(`${page.page_key} ${page.summary}`, queryText), })) .filter((page) => page.rrfScore > 0) .sort((a, b) => b.rrfScore - a.rrfScore || a.pageKey.localeCompare(b.pageKey)) .slice(0, limit); } private pagePath(scope: string, scopeId: string | null, pageKey: string): string { return scope === 'GLOBAL' ? `wiki/global/${pageKey}.md` : `wiki/user/${scopeId}/${pageKey}.md`; } } class NoopKnowledgeEventPort implements KnowledgeEventPort { async createEvent(): Promise {} } class NoopKnowledgeSlRefsPort implements MemoryKnowledgeSlRefsPort { async syncFromWiki(): Promise<{ inserted: number; deleted: number }> { return { inserted: 0, deleted: 0 }; } } class LocalMemoryConnections implements MemoryConnectionPort, SlConnectionCatalogPort { constructor( private readonly project: KtxLocalProject, private readonly queryExecutor?: { execute(input: { connectionId: string; sql: string; maxRows?: number }): Promise; }, ) {} async listEnabledConnections(ids: string[]): Promise { return ids .map((id) => localConnectionInfoFromConfig(id, this.project.config.connections[id])) .filter((connection): connection is KtxConnectionInfo => connection !== null); } async getConnectionById(connectionId: string): Promise { const connection = localConnectionInfoFromConfig(connectionId, this.project.config.connections[connectionId]); if (!connection) { throw new Error(`Connection not found: ${connectionId}`); } return connection; } async executeQuery(connectionId: string, sql: string): Promise { if (!this.queryExecutor) { throw new Error('Local memory capture has no query executor configured'); } return this.queryExecutor.execute({ connectionId, sql }); } } class LocalSlPythonPort implements SlPythonPort { constructor(private readonly compute?: KtxSemanticLayerComputePort) {} async validateSources(input: Parameters[0]) { if (!this.compute) { return { data: { errors: [], warnings: [LOCAL_SHAPE_WARNING], per_source_warnings: {}, }, }; } const result = await this.compute.validateSources({ sources: input.sources, dialect: input.dialect, recentlyTouched: input.recently_touched, }); return { data: { errors: result.errors, warnings: result.warnings, per_source_warnings: result.perSourceWarnings, }, }; } async query(input: Parameters[0]) { if (!this.compute) { return { error: 'Local memory capture has no semantic compute adapter configured' }; } const result = await this.compute.query({ sources: input.sources, dialect: input.dialect, query: input.query, }); return { data: { sql: result.sql, plan: result.plan } }; } } class LocalAuthorResolver implements GitAuthorResolverPort { async resolve() { return LOCAL_AUTHOR; } } class LocalMemoryLock { async withLock(_key: 'config:repo', fn: () => Promise): Promise { return fn(); } } class NoopSemanticLayerSourceReconciler implements MemorySlSourceReconcilerPort { async upsertRow(): Promise {} } class LocalShapeOnlySlValidator implements SlValidatorPort { async validateSingleSource(deps: SlValidationDeps, connectionId: string, sourceName: string) { try { const file = await deps.semanticLayerService.readSourceFile(connectionId, sourceName); const parsed = YAML.parse(file.content) as SemanticLayerSource; const isOverlay = parsed.table == null && parsed.sql == null; const result = (isOverlay ? sourceOverlaySchema : sourceDefinitionSchema).safeParse(parsed); return result.success ? { errors: [], warnings: [LOCAL_SHAPE_WARNING] } : { errors: result.error.issues.map( (issue) => `${sourceName}: ${issue.path.join('.') || 'source'} ${issue.message}`, ), warnings: [], }; } catch (error) { return { errors: [`${sourceName}: ${error instanceof Error ? error.message : String(error)}`], warnings: [] }; } } } class LocalMemoryToolSet implements MemoryToolSetLike { constructor(private readonly tools: BaseTool[]) {} toRuntimeTools(context: ToolContext): KtxRuntimeToolSet { return Object.fromEntries(this.tools.map((tool) => [tool.name, tool.toRuntimeTool(context)])); } } class LocalMemoryToolsetFactory implements MemoryToolsetFactoryPort { private readonly wikiTools: BaseTool[]; private readonly slTools: BaseTool[]; constructor(deps: { project: KtxLocalProject; embedding: KtxEmbeddingPort; wikiService: KnowledgeWikiService; knowledgeIndex: KnowledgeIndexPort; knowledgeEvents: KnowledgeEventPort; semanticLayerService: SemanticLayerService; slSearchService: SlSearchService; authorResolver: GitAuthorResolverPort; slSourcesRepository: SlSourcesIndexPort; connections: SlConnectionCatalogPort; }) { const slDeps = { semanticLayerService: deps.semanticLayerService, slSearchService: deps.slSearchService, authorResolver: deps.authorResolver, }; this.wikiTools = [ new WikiReadTool(deps.wikiService, deps.knowledgeIndex), new WikiSearchTool({ search: async (input) => { const results = await searchLocalKnowledgePages(deps.project, { userId: input.userId, query: input.query, limit: input.limit, embeddingService: deps.embedding, }); return { results: results.slice(0, input.limit).map((result) => ({ key: result.key, path: result.path, summary: result.summary, score: result.score, matchReasons: result.matchReasons, lanes: result.lanes, })), totalFound: results.length, }; }, }), new WikiListTagsTool(deps.knowledgeIndex), new WikiWriteTool(deps.wikiService, deps.knowledgeIndex, deps.knowledgeEvents), new WikiRemoveTool(deps.wikiService, deps.knowledgeIndex, deps.knowledgeEvents), ]; this.slTools = [ new SlDiscoverTool(slDeps, { maxSources: 25, minRrfScore: 0, maxDetailedSources: 5 }), new SlEditSourceTool(slDeps), new SlReadSourceTool(slDeps), new SlWriteSourceTool(slDeps), new SlValidateTool(slDeps), new SlRollbackTool(deps.slSourcesRepository, deps.connections, 0), ]; } createIngestWuToolset(): MemoryToolSetLike { return new LocalMemoryToolSet([...this.wikiTools, ...this.slTools]); } createToolset(): MemoryToolSetLike { return new LocalMemoryToolSet(this.wikiTools); } } function parseWiki(raw: string): { summary: string; content: string } { const match = raw.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/); if (!match) { return { summary: '', content: raw.trim() }; } const frontmatter = (YAML.parse(match[1]) ?? {}) as Record; return { summary: typeof frontmatter.summary === 'string' ? frontmatter.summary : '', content: match[2].trim(), }; } function parseWikiTags(raw: string): string[] { const match = raw.match(/^---\n([\s\S]*?)\n---\n?/); if (!match) { return []; } const frontmatter = (YAML.parse(match[1]) ?? {}) as Record; return Array.isArray(frontmatter.tags) ? frontmatter.tags.filter((tag): tag is string => typeof tag === 'string') : []; } function scoreText(text: string, query: string): number { const normalized = query.toLowerCase().trim(); if (!normalized) { return 0; } const haystack = text.toLowerCase(); if (haystack.includes(normalized)) { return 1; } const words = normalized.split(/\s+/).filter(Boolean); return words.filter((word) => haystack.includes(word)).length / Math.max(words.length, 1); }