fix(cli): resolve embedding provider explicitly and surface lane status in sl search (#192)

* feat(cli): add tryUseManagedLocalEmbeddingsDaemon for read-only callers

* feat(cli): add resolveProjectEmbeddingProvider helper

* fix(cli): wire sl search through resolveProjectEmbeddingProvider so semantic lane works

* fix(cli): wire wiki/knowledge search through resolveProjectEmbeddingProvider

* feat(cli): surface embeddings-unavailable status when sl search returns empty

* refactor(cli): route admin reindex through resolveProjectEmbeddingProvider

* refactor: pass embeddingProvider into ingest/scan instead of resolving inside @ktx/context

* refactor(mcp): resolve embedding provider in CLI factory, pass into context ports

* refactor(context): delete MANAGED_SENTENCE_TRANSFORMERS_BASE_URL sentinel

* refactor(cli): delete sentinel-based managed-embeddings indirection

* chore: scrub stale managed-embeddings sentinel references from tests and smoke script

* chore: unexport unused EmbeddingResolutionMode alias

* fix(cli): force pathPrefix="" when targeting the managed embeddings daemon

The managed daemon serves /embeddings/compute directly. The default
pathPrefix in @ktx/llm is /api, so omitting sentenceTransformers from
ktx.yaml produced /api/embeddings/compute -> 404. The resolver now
sets pathPrefix='' explicitly when wiring the managed daemon URL,
matching what the daemon actually exposes.
This commit is contained in:
Andrey Avtomonov 2026-05-21 02:21:22 +02:00 committed by GitHub
parent 56a967278a
commit 9d92c79988
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
36 changed files with 750 additions and 442 deletions

View file

@ -1,15 +1,11 @@
import {
createLocalKtxEmbeddingProviderFromConfig,
KtxIngestEmbeddingPortAdapter,
type KtxEmbeddingPort,
} from '@ktx/context';
import { KtxIngestEmbeddingPortAdapter, type KtxEmbeddingPort } from '@ktx/context';
import { reindexLocalIndexes, type ReindexScopeResult, type ReindexSummary } from '@ktx/context/index-sync';
import { type KtxLocalProject } from '@ktx/context/project';
import { loadKtxProject } from '@ktx/context/project';
import { Option, type Command } from '@commander-js/extra-typings';
import { cancel, intro, log, note, outro } from '@clack/prompts';
import type { KtxCliCommandContext } from './cli-program.js';
import { loadKtxCliProject } from './cli-project.js';
import type { KtxCliIo } from './cli-runtime.js';
import { resolveProjectEmbeddingProvider } from './embedding-resolution.js';
import { resolveOutputMode } from './io/mode.js';
import { green, red, SYMBOLS } from './io/symbols.js';
@ -48,15 +44,6 @@ export function registerAdminReindexCommand(admin: Command, context: KtxCliComma
});
}
function resolveReindexEmbeddingService(project: KtxLocalProject): KtxEmbeddingPort | null {
const config = project.config.ingest.embeddings;
if (config.backend === 'none') {
return null;
}
const provider = createLocalKtxEmbeddingProviderFromConfig(config);
return provider ? new KtxIngestEmbeddingPortAdapter(provider) : null;
}
function scopeKey(scope: ReindexScopeResult): string {
if (scope.kind === 'wiki') {
return scope.scope === 'user' ? `wiki/user/${scope.scopeId ?? 'local'}` : 'wiki/global';
@ -166,13 +153,16 @@ function renderReindexPretty(summary: ReindexSummary, io: KtxCliIo): void {
async function runKtxAdminReindex(args: KtxAdminReindexArgs, io: KtxCliIo = process): Promise<number> {
try {
const project = await loadKtxCliProject({
projectDir: args.projectDir,
const project = await loadKtxProject({ projectDir: args.projectDir });
const resolution = await resolveProjectEmbeddingProvider(project, {
mode: 'use-if-running',
cliVersion: args.cliVersion,
installPolicy: 'never',
io,
});
const embeddingService = resolveReindexEmbeddingService(project);
const embeddingService: KtxEmbeddingPort | null =
resolution.kind === 'configured' || resolution.kind === 'managed-running' || resolution.kind === 'managed-started'
? new KtxIngestEmbeddingPortAdapter(resolution.provider)
: null;
const summary = await reindexLocalIndexes(project, { force: args.force, embeddingService });
const mode = resolveOutputMode({ explicit: args.output, json: args.json, io });

View file

@ -1,29 +1,6 @@
import { describe, expect, it, vi } from 'vitest';
import { MANAGED_SENTENCE_TRANSFORMERS_BASE_URL } from '@ktx/context';
import { buildDefaultKtxProjectConfig, type KtxLocalProject, type KtxProjectConfig } from '@ktx/context/project';
import {
loadKtxCliProject,
projectNeedsManagedLocalEmbeddings,
substituteManagedLocalEmbeddingsUrl,
} from './cli-project.js';
import type { ManagedLocalEmbeddingsDaemon } from './managed-local-embeddings.js';
const RESOLVED_BASE_URL = 'http://127.0.0.1:51234';
function makeIo() {
let stderr = '';
return {
io: {
stdout: { write: (_chunk: string) => {} },
stderr: {
write: (chunk: string) => {
stderr += chunk;
},
},
},
stderr: () => stderr,
};
}
import { loadKtxCliProject } from './cli-project.js';
function projectWithConfig(config: KtxProjectConfig): KtxLocalProject {
return {
@ -36,147 +13,14 @@ function projectWithConfig(config: KtxProjectConfig): KtxLocalProject {
};
}
function withManagedIngestEmbedding(config: KtxProjectConfig): KtxProjectConfig {
return {
...config,
ingest: {
...config.ingest,
embeddings: {
backend: 'sentence-transformers',
model: 'all-MiniLM-L6-v2',
dimensions: 384,
sentenceTransformers: { base_url: MANAGED_SENTENCE_TRANSFORMERS_BASE_URL, pathPrefix: '' },
},
},
};
}
function withManagedScanEnrichmentEmbedding(config: KtxProjectConfig): KtxProjectConfig {
return {
...config,
scan: {
...config.scan,
enrichment: {
...config.scan.enrichment,
embeddings: {
backend: 'sentence-transformers',
model: 'all-MiniLM-L6-v2',
dimensions: 384,
sentenceTransformers: { base_url: MANAGED_SENTENCE_TRANSFORMERS_BASE_URL, pathPrefix: '' },
},
},
},
};
}
const fakeDaemon: ManagedLocalEmbeddingsDaemon = {
baseUrl: RESOLVED_BASE_URL,
stdoutLog: '/work/proj/.ktx/runtime/daemon.stdout.log',
stderrLog: '/work/proj/.ktx/runtime/daemon.stderr.log',
};
describe('projectNeedsManagedLocalEmbeddings', () => {
it('returns false when neither ingest nor scan embeddings reference the managed sentinel', () => {
expect(projectNeedsManagedLocalEmbeddings(buildDefaultKtxProjectConfig())).toBe(false);
});
it('returns true when ingest.embeddings uses the managed sentinel', () => {
expect(projectNeedsManagedLocalEmbeddings(withManagedIngestEmbedding(buildDefaultKtxProjectConfig()))).toBe(true);
});
it('returns true when scan.enrichment.embeddings uses the managed sentinel', () => {
expect(
projectNeedsManagedLocalEmbeddings(withManagedScanEnrichmentEmbedding(buildDefaultKtxProjectConfig())),
).toBe(true);
});
});
describe('substituteManagedLocalEmbeddingsUrl', () => {
it('rewrites the managed sentinel in both ingest.embeddings and scan.enrichment.embeddings', () => {
const config = withManagedScanEnrichmentEmbedding(withManagedIngestEmbedding(buildDefaultKtxProjectConfig()));
const resolved = substituteManagedLocalEmbeddingsUrl(config, RESOLVED_BASE_URL);
expect(resolved.ingest.embeddings.sentenceTransformers?.base_url).toBe(RESOLVED_BASE_URL);
expect(resolved.scan.enrichment.embeddings?.sentenceTransformers?.base_url).toBe(RESOLVED_BASE_URL);
});
it('returns the input unchanged when no sentinel is present', () => {
const config = buildDefaultKtxProjectConfig();
const resolved = substituteManagedLocalEmbeddingsUrl(config, RESOLVED_BASE_URL);
expect(resolved.ingest.embeddings).toEqual(config.ingest.embeddings);
expect(resolved.scan.enrichment.embeddings).toEqual(config.scan.enrichment.embeddings);
});
it('does not touch non-sentinel sentence-transformers URLs', () => {
const config: KtxProjectConfig = {
...buildDefaultKtxProjectConfig(),
ingest: {
...buildDefaultKtxProjectConfig().ingest,
embeddings: {
backend: 'sentence-transformers',
model: 'all-MiniLM-L6-v2',
dimensions: 384,
sentenceTransformers: { base_url: 'http://localhost:9999', pathPrefix: '' },
},
},
};
const resolved = substituteManagedLocalEmbeddingsUrl(config, RESOLVED_BASE_URL);
expect(resolved.ingest.embeddings.sentenceTransformers?.base_url).toBe('http://localhost:9999');
});
});
describe('loadKtxCliProject', () => {
it('returns the project unchanged and does not start the daemon when no sentinel is present', async () => {
const io = makeIo();
it('delegates to loadKtxProject and returns the project unchanged', async () => {
const project = projectWithConfig(buildDefaultKtxProjectConfig());
const loadProject = vi.fn(async () => project);
const ensureLocalEmbeddings = vi.fn(async () => fakeDaemon);
const result = await loadKtxCliProject(
{ projectDir: '/work/proj', cliVersion: '0.2.0', installPolicy: 'never', io: io.io },
{ loadProject, ensureLocalEmbeddings },
);
const result = await loadKtxCliProject({ projectDir: '/work/proj' }, { loadProject });
expect(result).toBe(project);
expect(ensureLocalEmbeddings).not.toHaveBeenCalled();
});
it('starts the daemon and substitutes the resolved URL when ingest.embeddings uses the sentinel', async () => {
const io = makeIo();
const project = projectWithConfig(withManagedIngestEmbedding(buildDefaultKtxProjectConfig()));
const loadProject = vi.fn(async () => project);
const ensureLocalEmbeddings = vi.fn(async () => fakeDaemon);
const result = await loadKtxCliProject(
{ projectDir: '/work/proj', cliVersion: '0.2.0', installPolicy: 'never', io: io.io },
{ loadProject, ensureLocalEmbeddings },
);
expect(ensureLocalEmbeddings).toHaveBeenCalledWith({
cliVersion: '0.2.0',
projectDir: '/work/proj',
installPolicy: 'never',
io: io.io,
});
expect(result.config.ingest.embeddings.sentenceTransformers?.base_url).toBe(RESOLVED_BASE_URL);
});
it('does not mutate process.env', async () => {
const io = makeIo();
const before = process.env.KTX_MANAGED_SENTENCE_TRANSFORMERS_BASE_URL;
delete process.env.KTX_MANAGED_SENTENCE_TRANSFORMERS_BASE_URL;
try {
const project = projectWithConfig(withManagedIngestEmbedding(buildDefaultKtxProjectConfig()));
await loadKtxCliProject(
{ projectDir: '/work/proj', cliVersion: '0.2.0', installPolicy: 'never', io: io.io },
{ loadProject: vi.fn(async () => project), ensureLocalEmbeddings: vi.fn(async () => fakeDaemon) },
);
expect(process.env.KTX_MANAGED_SENTENCE_TRANSFORMERS_BASE_URL).toBeUndefined();
} finally {
if (before === undefined) {
delete process.env.KTX_MANAGED_SENTENCE_TRANSFORMERS_BASE_URL;
} else {
process.env.KTX_MANAGED_SENTENCE_TRANSFORMERS_BASE_URL = before;
}
}
expect(loadProject).toHaveBeenCalledWith({ projectDir: '/work/proj' });
});
});

View file

@ -1,91 +1,20 @@
import { MANAGED_SENTENCE_TRANSFORMERS_BASE_URL } from '@ktx/context';
import { loadKtxProject, type KtxLocalProject } from '@ktx/context/project';
import type { KtxProjectConfig, KtxProjectEmbeddingConfig } from '@ktx/context/project';
import type { KtxCliIo } from './cli-runtime.js';
import {
ensureManagedLocalEmbeddingsDaemon,
type ManagedLocalEmbeddingsDaemon,
} from './managed-local-embeddings.js';
import type { KtxManagedPythonInstallPolicy } from './managed-python-command.js';
export interface LoadKtxCliProjectOptions {
projectDir: string;
cliVersion: string;
installPolicy: KtxManagedPythonInstallPolicy;
io: KtxCliIo;
}
export interface LoadKtxCliProjectDeps {
loadProject?: typeof loadKtxProject;
ensureLocalEmbeddings?: (
options: Parameters<typeof ensureManagedLocalEmbeddingsDaemon>[0],
) => Promise<ManagedLocalEmbeddingsDaemon>;
}
/**
* Thin wrapper around `loadKtxProject`. Kept as a single entrypoint so the CLI can grow shared
* pre-load behavior later (telemetry, project lock, etc.). Today it does no extra work.
*/
export async function loadKtxCliProject(
options: LoadKtxCliProjectOptions,
deps: LoadKtxCliProjectDeps = {},
): Promise<KtxLocalProject> {
const loadProject = deps.loadProject ?? loadKtxProject;
const ensureLocalEmbeddings = deps.ensureLocalEmbeddings ?? ensureManagedLocalEmbeddingsDaemon;
const project = await loadProject({ projectDir: options.projectDir });
if (!projectNeedsManagedLocalEmbeddings(project.config)) {
return project;
}
const daemon = await ensureLocalEmbeddings({
cliVersion: options.cliVersion,
projectDir: options.projectDir,
installPolicy: options.installPolicy,
io: options.io,
});
return {
...project,
config: substituteManagedLocalEmbeddingsUrl(project.config, daemon.baseUrl),
};
}
export function projectNeedsManagedLocalEmbeddings(config: KtxProjectConfig): boolean {
return (
embeddingUsesManagedSentinel(config.ingest.embeddings) ||
embeddingUsesManagedSentinel(config.scan.enrichment.embeddings)
);
}
export function substituteManagedLocalEmbeddingsUrl(
config: KtxProjectConfig,
baseUrl: string,
): KtxProjectConfig {
const ingestEmbeddings = rewriteManagedEmbeddingConfig(config.ingest.embeddings, baseUrl);
const scanEnrichmentEmbeddings = rewriteManagedEmbeddingConfig(config.scan.enrichment.embeddings, baseUrl);
return {
...config,
ingest: { ...config.ingest, embeddings: ingestEmbeddings },
scan: {
...config.scan,
enrichment: { ...config.scan.enrichment, embeddings: scanEnrichmentEmbeddings },
},
};
}
function embeddingUsesManagedSentinel(embedding: KtxProjectEmbeddingConfig | undefined): boolean {
return embedding?.sentenceTransformers?.base_url === MANAGED_SENTENCE_TRANSFORMERS_BASE_URL;
}
function rewriteManagedEmbeddingConfig<T extends KtxProjectEmbeddingConfig | undefined>(
embedding: T,
baseUrl: string,
): T {
if (!embedding || !embeddingUsesManagedSentinel(embedding)) {
return embedding;
}
return {
...embedding,
sentenceTransformers: {
...embedding.sentenceTransformers,
base_url: baseUrl,
},
} as T;
return (deps.loadProject ?? loadKtxProject)({ projectDir: options.projectDir });
}

View file

@ -59,6 +59,7 @@ export function registerWikiCommands(program: Command, context: KtxCliCommandCon
userId: options.userId,
output: options.output,
json: options.json,
cliVersion: context.packageInfo.version,
});
return;
}
@ -71,6 +72,7 @@ export function registerWikiCommands(program: Command, context: KtxCliCommandCon
json: options.json,
...(isDebugEnabled(command) ? { debug: true } : {}),
...(options.limit !== undefined ? { limit: options.limit } : {}),
cliVersion: context.packageInfo.version,
});
},
);

View file

@ -77,6 +77,7 @@ export function registerSlCommands(program: Command, context: KtxCliCommandConte
connectionId: options.connectionId,
output: options.output,
json: options.json,
cliVersion: context.packageInfo.version,
});
return;
}
@ -88,6 +89,7 @@ export function registerSlCommands(program: Command, context: KtxCliCommandConte
...(options.limit !== undefined ? { limit: options.limit } : {}),
output: options.output,
json: options.json,
cliVersion: context.packageInfo.version,
});
},
);

View file

@ -0,0 +1,145 @@
import { describe, expect, it, vi } from 'vitest';
import { buildDefaultKtxProjectConfig, type KtxLocalProject, type KtxProjectConfig } from '@ktx/context/project';
import { resolveProjectEmbeddingProvider } from './embedding-resolution.js';
import type { ManagedLocalEmbeddingsDaemon } from './managed-local-embeddings.js';
function projectWithConfig(config: KtxProjectConfig): KtxLocalProject {
return {
projectDir: '/work/proj',
configPath: '/work/proj/ktx.yaml',
config,
coreConfig: {} as KtxLocalProject['coreConfig'],
git: {} as KtxLocalProject['git'],
fileStore: {} as KtxLocalProject['fileStore'],
};
}
function withManagedEmbedding(config: KtxProjectConfig, base_url?: string): KtxProjectConfig {
return {
...config,
ingest: {
...config.ingest,
embeddings: {
backend: 'sentence-transformers',
model: 'all-MiniLM-L6-v2',
dimensions: 384,
...(base_url === undefined
? {}
: { sentenceTransformers: { base_url, pathPrefix: '' } }),
},
},
};
}
const noopIo = {
stdout: { write: (_chunk: string) => {} },
stderr: { write: (_chunk: string) => {} },
} as const;
const fakeDaemon: ManagedLocalEmbeddingsDaemon = {
baseUrl: 'http://127.0.0.1:51234',
stdoutLog: '/tmp/o',
stderrLog: '/tmp/e',
};
describe('resolveProjectEmbeddingProvider', () => {
it('returns disabled when backend is none', async () => {
const project = projectWithConfig(buildDefaultKtxProjectConfig());
const result = await resolveProjectEmbeddingProvider(project, {
mode: 'use-if-running',
cliVersion: '0.5.0',
io: noopIo,
});
expect(result.kind).toBe('disabled');
});
it('returns a configured provider when base_url is explicit', async () => {
const project = projectWithConfig(withManagedEmbedding(buildDefaultKtxProjectConfig(), 'http://my-st:8080'));
const createKtxEmbeddingProvider = vi.fn(() => ({ id: 'fake' }) as never);
const result = await resolveProjectEmbeddingProvider(project, {
mode: 'use-if-running',
cliVersion: '0.5.0',
io: noopIo,
createKtxEmbeddingProvider,
});
expect(result.kind).toBe('configured');
expect(createKtxEmbeddingProvider).toHaveBeenCalledOnce();
});
it('connects to the running managed daemon when base_url is omitted', async () => {
const project = projectWithConfig(withManagedEmbedding(buildDefaultKtxProjectConfig(), undefined));
const tryUseManaged = vi.fn(async () => fakeDaemon);
const createKtxEmbeddingProvider = vi.fn(() => ({ id: 'fake' }) as never);
const ensureManaged = vi.fn(async () => fakeDaemon);
const result = await resolveProjectEmbeddingProvider(project, {
mode: 'use-if-running',
cliVersion: '0.5.0',
io: noopIo,
createKtxEmbeddingProvider,
tryUseManagedDaemon: tryUseManaged,
ensureManagedDaemon: ensureManaged,
});
expect(result.kind).toBe('managed-running');
expect(tryUseManaged).toHaveBeenCalledOnce();
expect(ensureManaged).not.toHaveBeenCalled();
});
it('passes pathPrefix="" to the embedding provider when targeting the managed daemon', async () => {
const project = projectWithConfig(withManagedEmbedding(buildDefaultKtxProjectConfig(), undefined));
const tryUseManaged = vi.fn(async () => fakeDaemon);
const createKtxEmbeddingProvider = vi.fn(() => ({ id: 'fake' }) as never);
await resolveProjectEmbeddingProvider(project, {
mode: 'use-if-running',
cliVersion: '0.5.0',
io: noopIo,
createKtxEmbeddingProvider,
tryUseManagedDaemon: tryUseManaged,
});
expect(createKtxEmbeddingProvider).toHaveBeenCalledWith(
expect.objectContaining({
sentenceTransformers: expect.objectContaining({
baseURL: fakeDaemon.baseUrl,
pathPrefix: '',
}),
}),
);
});
it('returns managed-unavailable when no daemon is running and mode is use-if-running', async () => {
const project = projectWithConfig(withManagedEmbedding(buildDefaultKtxProjectConfig(), ''));
const tryUseManaged = vi.fn(async () => null);
const ensureManaged = vi.fn(async () => fakeDaemon);
const result = await resolveProjectEmbeddingProvider(project, {
mode: 'use-if-running',
cliVersion: '0.5.0',
io: noopIo,
tryUseManagedDaemon: tryUseManaged,
ensureManagedDaemon: ensureManaged,
});
expect(result.kind).toBe('managed-unavailable');
expect(ensureManaged).not.toHaveBeenCalled();
});
it('starts the managed daemon when mode is ensure', async () => {
const project = projectWithConfig(withManagedEmbedding(buildDefaultKtxProjectConfig(), undefined));
const tryUseManaged = vi.fn(async () => null);
const ensureManaged = vi.fn(async () => fakeDaemon);
const createKtxEmbeddingProvider = vi.fn(() => ({ id: 'fake' }) as never);
const result = await resolveProjectEmbeddingProvider(project, {
mode: 'ensure',
installPolicy: 'auto',
cliVersion: '0.5.0',
io: noopIo,
createKtxEmbeddingProvider,
tryUseManagedDaemon: tryUseManaged,
ensureManagedDaemon: ensureManaged,
});
expect(result.kind).toBe('managed-started');
expect(ensureManaged).toHaveBeenCalledWith({
cliVersion: '0.5.0',
projectDir: '/work/proj',
installPolicy: 'auto',
io: noopIo,
});
});
});

View file

@ -0,0 +1,107 @@
import {
type KtxEmbeddingProvider,
createKtxEmbeddingProvider as defaultCreateKtxEmbeddingProvider,
} from '@ktx/llm';
import type { KtxLocalProject, KtxProjectEmbeddingConfig } from '@ktx/context/project';
import { resolveLocalKtxEmbeddingConfig } from '@ktx/context';
import type { KtxCliIo } from './cli-runtime.js';
import {
ensureManagedLocalEmbeddingsDaemon as defaultEnsureManagedDaemon,
tryUseManagedLocalEmbeddingsDaemon as defaultTryUseManagedDaemon,
} from './managed-local-embeddings.js';
import type { KtxManagedPythonInstallPolicy } from './managed-python-command.js';
type EmbeddingResolutionMode = 'ensure' | 'use-if-running';
export type EmbeddingProviderResolution =
| { kind: 'disabled' }
| { kind: 'configured'; provider: KtxEmbeddingProvider; baseUrl: string }
| { kind: 'managed-running'; provider: KtxEmbeddingProvider; baseUrl: string }
| { kind: 'managed-started'; provider: KtxEmbeddingProvider; baseUrl: string }
| { kind: 'managed-unavailable'; reason: string };
export interface ResolveProjectEmbeddingProviderOptions {
mode: EmbeddingResolutionMode;
cliVersion: string;
io: KtxCliIo;
/** Required when mode === 'ensure'. */
installPolicy?: KtxManagedPythonInstallPolicy;
tryUseManagedDaemon?: typeof defaultTryUseManagedDaemon;
ensureManagedDaemon?: typeof defaultEnsureManagedDaemon;
createKtxEmbeddingProvider?: typeof defaultCreateKtxEmbeddingProvider;
}
function usesManagedDaemon(embeddings: KtxProjectEmbeddingConfig): boolean {
if (embeddings.backend !== 'sentence-transformers') {
return false;
}
const baseUrl = embeddings.sentenceTransformers?.base_url;
return baseUrl === undefined || baseUrl === '';
}
export async function resolveProjectEmbeddingProvider(
project: KtxLocalProject,
options: ResolveProjectEmbeddingProviderOptions,
): Promise<EmbeddingProviderResolution> {
const embeddings = project.config.ingest.embeddings;
if (embeddings.backend === 'none') {
return { kind: 'disabled' };
}
const createProvider = options.createKtxEmbeddingProvider ?? defaultCreateKtxEmbeddingProvider;
if (!usesManagedDaemon(embeddings)) {
const resolved = resolveLocalKtxEmbeddingConfig(embeddings, process.env);
if (!resolved) {
return { kind: 'managed-unavailable', reason: 'embedding config missing required fields' };
}
const provider = createProvider(resolved);
const baseUrl = embeddings.sentenceTransformers?.base_url ?? '';
return { kind: 'configured', provider, baseUrl };
}
const tryUse = options.tryUseManagedDaemon ?? defaultTryUseManagedDaemon;
const running = await tryUse({ cliVersion: options.cliVersion, projectDir: project.projectDir });
if (running) {
const provider = buildManagedProvider(embeddings, running.baseUrl, createProvider);
return provider
? { kind: 'managed-running', provider, baseUrl: running.baseUrl }
: { kind: 'managed-unavailable', reason: 'failed to build embedding provider from running daemon' };
}
if (options.mode === 'use-if-running') {
return { kind: 'managed-unavailable', reason: 'managed embeddings daemon is not running' };
}
const ensure = options.ensureManagedDaemon ?? defaultEnsureManagedDaemon;
if (!options.installPolicy) {
throw new Error("installPolicy is required when mode === 'ensure'");
}
const daemon = await ensure({
cliVersion: options.cliVersion,
projectDir: project.projectDir,
installPolicy: options.installPolicy,
io: options.io,
});
const provider = buildManagedProvider(embeddings, daemon.baseUrl, createProvider);
return provider
? { kind: 'managed-started', provider, baseUrl: daemon.baseUrl }
: { kind: 'managed-unavailable', reason: 'failed to build embedding provider after starting daemon' };
}
function buildManagedProvider(
embeddings: KtxProjectEmbeddingConfig,
baseUrl: string,
createProvider: typeof defaultCreateKtxEmbeddingProvider,
): KtxEmbeddingProvider | null {
const merged: KtxProjectEmbeddingConfig = {
...embeddings,
sentenceTransformers: {
...embeddings.sentenceTransformers,
base_url: baseUrl,
pathPrefix: '',
},
};
const resolved = resolveLocalKtxEmbeddingConfig(merged, process.env);
return resolved ? createProvider(resolved) : null;
}

View file

@ -152,12 +152,12 @@ describe('runKtxCli', () => {
await expect(runKtxCli(['--project-dir', tempDir, 'wiki', '--json'], listIo.io, { knowledge }))
.resolves.toBe(0);
expect(knowledge).toHaveBeenCalledWith(
{
expect.objectContaining({
command: 'list',
projectDir: tempDir,
userId: 'local',
json: true,
},
}),
listIo.io,
);
@ -166,14 +166,14 @@ describe('runKtxCli', () => {
runKtxCli(['--project-dir', tempDir, 'wiki', 'revenue', '--limit', '5'], searchIo.io, { knowledge }),
).resolves.toBe(0);
expect(knowledge).toHaveBeenLastCalledWith(
{
expect.objectContaining({
command: 'search',
projectDir: tempDir,
query: 'revenue',
userId: 'local',
json: false,
limit: 5,
},
}),
searchIo.io,
);
@ -182,14 +182,14 @@ describe('runKtxCli', () => {
runKtxCli(['--project-dir', tempDir, '--debug', 'wiki', 'revenue'], debugSearchIo.io, { knowledge }),
).resolves.toBe(0);
expect(knowledge).toHaveBeenLastCalledWith(
{
expect.objectContaining({
command: 'search',
projectDir: tempDir,
query: 'revenue',
userId: 'local',
json: false,
debug: true,
},
}),
debugSearchIo.io,
);
@ -198,13 +198,13 @@ describe('runKtxCli', () => {
runKtxCli(['--project-dir', tempDir, 'wiki', 'revenue', 'policy'], multiWordIo.io, { knowledge }),
).resolves.toBe(0);
expect(knowledge).toHaveBeenLastCalledWith(
{
expect.objectContaining({
command: 'search',
projectDir: tempDir,
query: 'revenue policy',
userId: 'local',
json: false,
},
}),
multiWordIo.io,
);
});
@ -248,7 +248,7 @@ describe('runKtxCli', () => {
),
).resolves.toBe(0);
expect(sl).toHaveBeenCalledWith(
{
expect.objectContaining({
command: 'search',
projectDir: tempDir,
connectionId: 'warehouse',
@ -256,7 +256,7 @@ describe('runKtxCli', () => {
limit: 5,
json: true,
output: undefined,
},
}),
searchIo.io,
);
@ -265,13 +265,13 @@ describe('runKtxCli', () => {
runKtxCli(['--project-dir', tempDir, 'sl', '--connection-id', 'warehouse', '--json'], bareIo.io, { sl }),
).resolves.toBe(0);
expect(sl).toHaveBeenLastCalledWith(
{
expect.objectContaining({
command: 'list',
projectDir: tempDir,
connectionId: 'warehouse',
json: true,
output: undefined,
},
}),
bareIo.io,
);

View file

@ -54,7 +54,6 @@ export type {
export {
ensureManagedLocalEmbeddingsDaemon,
managedLocalEmbeddingHealthConfig,
managedLocalEmbeddingProjectConfig,
type ManagedLocalEmbeddingsDaemon,
type ManagedLocalEmbeddingsOptions,
} from './managed-local-embeddings.js';

View file

@ -18,8 +18,8 @@ import {
sanitizeMemoryFlowError,
} from '@ktx/context/ingest';
import type { KtxSqlQueryExecutorPort } from '@ktx/context/connections';
import { type KtxLocalProject } from '@ktx/context/project';
import { loadKtxCliProject } from './cli-project.js';
import { loadKtxProject, type KtxLocalProject } from '@ktx/context/project';
import { resolveProjectEmbeddingProvider } from './embedding-resolution.js';
import { createKtxCliIngestQueryExecutor } from './ingest-query-executor.js';
import { readIngestReportSnapshotFile } from './ingest-report-file.js';
import { createCliOperationalLogger } from './io/logger.js';
@ -682,16 +682,17 @@ export async function runKtxIngest(
deps: KtxIngestDeps = {},
): Promise<number> {
try {
const cliVersion = args.command === 'run' ? args.cliVersion : undefined;
const runtimeInstallPolicy = args.command === 'run' ? args.runtimeInstallPolicy : undefined;
const project = await loadKtxCliProject({
projectDir: args.projectDir,
cliVersion: cliVersion ?? '0.0.0-private',
installPolicy: runtimeInstallPolicy ?? 'never',
io,
});
const project = await loadKtxProject({ projectDir: args.projectDir });
const env = deps.env ?? process.env;
if (args.command === 'run') {
const resolution = await resolveProjectEmbeddingProvider(project, {
mode: 'ensure',
installPolicy: args.runtimeInstallPolicy ?? 'never',
cliVersion: args.cliVersion ?? '0.0.0-private',
io,
});
const embeddingProvider =
resolution.kind === 'disabled' || resolution.kind === 'managed-unavailable' ? null : resolution.provider;
const ingestProject =
args.allowImplicitAdapter && !project.config.ingest.adapters.includes(args.adapter)
? {
@ -771,6 +772,7 @@ export async function runKtxIngest(
queryExecutor,
trigger: 'manual_resync',
jobIdFactory: deps.jobIdFactory,
embeddingProvider,
...(memoryFlow ? { memoryFlow } : {}),
...(progress ? { progress } : {}),
});
@ -843,6 +845,7 @@ export async function runKtxIngest(
...localIngestOptions,
queryExecutor,
pullConfigOptions: adapterOptions,
embeddingProvider,
...(args.debugLlmRequestFile ? { llmDebugRequestFile: args.debugLlmRequestFile } : {}),
...(memoryFlow ? { memoryFlow } : {}),
});

View file

@ -5,7 +5,7 @@ import { stripVTControlCharacters } from 'node:util';
import { initKtxProject, loadKtxProject } from '@ktx/context/project';
import type { KtxEmbeddingPort } from '@ktx/context';
import { writeLocalKnowledgePage } from '@ktx/context/wiki';
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { runKtxKnowledge } from './knowledge.js';
function makeIo() {
@ -81,12 +81,17 @@ describe('runKtxKnowledge', () => {
await seedWikiPage(projectDir);
const listIo = makeIo();
await expect(runKtxKnowledge({ command: 'list', projectDir, userId: 'local' }, listIo.io)).resolves.toBe(0);
await expect(
runKtxKnowledge({ command: 'list', projectDir, userId: 'local', cliVersion: '0.0.0-test' }, listIo.io),
).resolves.toBe(0);
expect(listIo.stdout()).toContain('GLOBAL\tmetrics-revenue\tRevenue');
const searchIo = makeIo();
await expect(
runKtxKnowledge({ command: 'search', projectDir, query: 'paid order', userId: 'local' }, searchIo.io),
runKtxKnowledge(
{ command: 'search', projectDir, query: 'paid order', userId: 'local', cliVersion: '0.0.0-test' },
searchIo.io,
),
).resolves.toBe(0);
expect(searchIo.stdout()).toContain('metrics-revenue');
});
@ -99,7 +104,14 @@ describe('runKtxKnowledge', () => {
const searchIo = makeIo();
await expect(
runKtxKnowledge(
{ command: 'search', projectDir, query: 'paid order', userId: 'local', output: 'pretty' },
{
command: 'search',
projectDir,
query: 'paid order',
userId: 'local',
output: 'pretty',
cliVersion: '0.0.0-test',
},
searchIo.io,
),
).resolves.toBe(0);
@ -115,9 +127,12 @@ describe('runKtxKnowledge', () => {
await seedWikiPage(projectDir);
const listIo = makeIo();
await expect(runKtxKnowledge({ command: 'list', projectDir, userId: 'local', json: true }, listIo.io)).resolves.toBe(
0,
);
await expect(
runKtxKnowledge(
{ command: 'list', projectDir, userId: 'local', json: true, cliVersion: '0.0.0-test' },
listIo.io,
),
).resolves.toBe(0);
expect(JSON.parse(listIo.stdout())).toMatchObject({
kind: 'list',
data: { items: [expect.objectContaining({ key: 'metrics-revenue', summary: 'Revenue' })] },
@ -127,7 +142,15 @@ describe('runKtxKnowledge', () => {
const searchIo = makeIo();
await expect(
runKtxKnowledge(
{ command: 'search', projectDir, query: 'paid order', userId: 'local', json: true, limit: 5 },
{
command: 'search',
projectDir,
query: 'paid order',
userId: 'local',
json: true,
limit: 5,
cliVersion: '0.0.0-test',
},
searchIo.io,
),
).resolves.toBe(0);
@ -144,7 +167,10 @@ describe('runKtxKnowledge', () => {
const searchIo = makeIo();
await expect(
runKtxKnowledge({ command: 'search', projectDir, query: 'revenue', userId: 'local' }, searchIo.io),
runKtxKnowledge(
{ command: 'search', projectDir, query: 'revenue', userId: 'local', cliVersion: '0.0.0-test' },
searchIo.io,
),
).resolves.toBe(0);
expect(searchIo.stdout()).toBe('');
@ -166,7 +192,7 @@ describe('runKtxKnowledge', () => {
const searchIo = makeIo();
await expect(
runKtxKnowledge(
{ command: 'search', projectDir, query: 'revenue', userId: 'local' },
{ command: 'search', projectDir, query: 'revenue', userId: 'local', cliVersion: '0.0.0-test' },
searchIo.io,
{ embeddingService: new FakeEmbeddingPort() },
),
@ -176,6 +202,37 @@ describe('runKtxKnowledge', () => {
expect(searchIo.stderr()).toBe('');
});
it('routes wiki search through resolveEmbeddingProvider when no embeddingService is injected', async () => {
const projectDir = join(tempDir, 'resolver-project');
await initKtxProject({ projectDir });
const search = vi.fn(async () => []);
const searchIo = makeIo();
await expect(
runKtxKnowledge(
{
command: 'search',
projectDir,
query: 'income',
userId: 'local',
cliVersion: '0.5.0',
},
searchIo.io,
{
resolveEmbeddingProvider: async () => ({
kind: 'managed-running',
provider: { id: 'fake' } as never,
baseUrl: 'http://127.0.0.1:51234',
}),
searchLocalKnowledgePages: search,
},
),
).resolves.toBe(0);
expect(search).toHaveBeenCalledWith(
expect.anything(),
expect.objectContaining({ embeddingService: expect.any(Object) }),
);
});
it('writes wiki search lane diagnostics to stderr when debug is enabled', async () => {
const projectDir = join(tempDir, 'debug-project');
await initKtxProject({ projectDir });
@ -184,7 +241,15 @@ describe('runKtxKnowledge', () => {
const searchIo = makeIo();
await expect(
runKtxKnowledge(
{ command: 'search', projectDir, query: 'paid order', userId: 'local', json: true, debug: true },
{
command: 'search',
projectDir,
query: 'paid order',
userId: 'local',
json: true,
debug: true,
cliVersion: '0.0.0-test',
},
searchIo.io,
{ embeddingService: new FakeEmbeddingPort() },
),

View file

@ -1,20 +1,20 @@
import {
createLocalKtxEmbeddingProviderFromConfig,
KtxIngestEmbeddingPortAdapter,
type KtxEmbeddingPort,
} from '@ktx/context';
import { KtxIngestEmbeddingPortAdapter, type KtxEmbeddingPort } from '@ktx/context';
import { loadKtxProject } from '@ktx/context/project';
import {
type LocalKnowledgeSearchResult,
type LocalKnowledgeSummary,
listLocalKnowledgePages,
searchLocalKnowledgePages,
searchLocalKnowledgePages as defaultSearchLocalKnowledgePages,
} from '@ktx/context/wiki';
import {
resolveProjectEmbeddingProvider,
type EmbeddingProviderResolution,
} from './embedding-resolution.js';
import { resolveOutputMode } from './io/mode.js';
import { createRankBadgeFormatter, printList, type PrintListColumn } from './io/print-list.js';
export type KtxKnowledgeArgs =
| { command: 'list'; projectDir: string; userId: string; output?: string; json?: boolean }
| { command: 'list'; projectDir: string; userId: string; output?: string; json?: boolean; cliVersion: string }
| {
command: 'search';
projectDir: string;
@ -24,6 +24,7 @@ export type KtxKnowledgeArgs =
json?: boolean;
limit?: number;
debug?: boolean;
cliVersion: string;
};
type KtxKnowledgeIo = import('./cli-runtime.js').KtxCliIo;
@ -54,20 +55,36 @@ function wikiSearchColumns(
interface KtxKnowledgeDeps {
embeddingService?: KtxEmbeddingPort | null;
createEmbeddingProvider?: typeof createLocalKtxEmbeddingProviderFromConfig;
resolveEmbeddingProvider?: typeof resolveProjectEmbeddingProvider;
searchLocalKnowledgePages?: typeof defaultSearchLocalKnowledgePages;
}
function wikiSearchEmbeddingService(
function resolutionToEmbeddingPort(resolution: EmbeddingProviderResolution): KtxEmbeddingPort | null {
if (
resolution.kind === 'configured' ||
resolution.kind === 'managed-running' ||
resolution.kind === 'managed-started'
) {
return new KtxIngestEmbeddingPortAdapter(resolution.provider);
}
return null;
}
async function wikiSearchEmbeddingService(
project: Awaited<ReturnType<typeof loadKtxProject>>,
deps: KtxKnowledgeDeps,
): KtxEmbeddingPort | null {
args: { cliVersion: string },
io: KtxKnowledgeIo,
): Promise<KtxEmbeddingPort | null> {
if ('embeddingService' in deps) {
return deps.embeddingService ?? null;
}
const provider = (deps.createEmbeddingProvider ?? createLocalKtxEmbeddingProviderFromConfig)(
project.config.ingest.embeddings,
);
return provider ? new KtxIngestEmbeddingPortAdapter(provider) : null;
const resolution = await (deps.resolveEmbeddingProvider ?? resolveProjectEmbeddingProvider)(project, {
mode: 'use-if-running',
cliVersion: args.cliVersion,
io,
});
return resolutionToEmbeddingPort(resolution);
}
function writeWikiSearchDebug(
@ -114,8 +131,9 @@ export async function runKtxKnowledge(
return 0;
}
if (args.command === 'search') {
const embeddingService = wikiSearchEmbeddingService(project, deps);
const results = await searchLocalKnowledgePages(project, {
const embeddingService = await wikiSearchEmbeddingService(project, deps, { cliVersion: args.cliVersion }, io);
const search = deps.searchLocalKnowledgePages ?? defaultSearchLocalKnowledgePages;
const results = await search(project, {
query: args.query,
userId: args.userId,
embeddingService,

View file

@ -1,12 +1,12 @@
import { describe, expect, it, vi } from 'vitest';
import { MANAGED_SENTENCE_TRANSFORMERS_BASE_URL } from '@ktx/context';
import {
ensureManagedLocalEmbeddingsDaemon,
managedLocalEmbeddingHealthConfig,
managedLocalEmbeddingProjectConfig,
tryUseManagedLocalEmbeddingsDaemon,
} from './managed-local-embeddings.js';
import type { ManagedPythonCommandRuntime } from './managed-python-command.js';
import type { ManagedPythonDaemonStartResult } from './managed-python-daemon.js';
import type { ManagedPythonDaemonLayout } from './managed-python-runtime.js';
function makeIo() {
let stdout = '';
@ -94,25 +94,6 @@ function daemonResult(status: 'started' | 'reused' = 'reused'): ManagedPythonDae
};
}
describe('managedLocalEmbeddingProjectConfig', () => {
it('uses a stable managed runtime marker instead of a random daemon port', () => {
expect(
managedLocalEmbeddingProjectConfig({
model: 'all-MiniLM-L6-v2',
dimensions: 384,
}),
).toEqual({
backend: 'sentence-transformers',
model: 'all-MiniLM-L6-v2',
dimensions: 384,
sentenceTransformers: {
base_url: MANAGED_SENTENCE_TRANSFORMERS_BASE_URL,
pathPrefix: '',
},
});
});
});
describe('managedLocalEmbeddingHealthConfig', () => {
it('uses the active KTX daemon URL for the immediate health check', () => {
expect(
@ -181,3 +162,93 @@ describe('ensureManagedLocalEmbeddingsDaemon', () => {
expect(io.stderr()).toContain('Using KTX daemon: http://127.0.0.1:61234');
});
});
describe('tryUseManagedLocalEmbeddingsDaemon', () => {
it('returns the daemon when one is running and healthy', async () => {
const readStatus = vi.fn(async () => ({
kind: 'running' as const,
detail: 'ok',
layout: {} as ManagedPythonDaemonLayout,
state: {
schemaVersion: 1 as const,
pid: 123,
host: '127.0.0.1' as const,
port: 4321,
version: '0.5.0',
features: ['local-embeddings' as const],
startedAt: '2026-05-21T00:00:00Z',
stdoutLog: '/tmp/stdout.log',
stderrLog: '/tmp/stderr.log',
},
baseUrl: 'http://127.0.0.1:4321',
}));
const result = await tryUseManagedLocalEmbeddingsDaemon({
cliVersion: '0.5.0',
projectDir: '/work/proj',
readStatus,
});
expect(result).toEqual({
baseUrl: 'http://127.0.0.1:4321',
stdoutLog: '/tmp/stdout.log',
stderrLog: '/tmp/stderr.log',
});
expect(readStatus).toHaveBeenCalledWith({
cliVersion: '0.5.0',
projectDir: '/work/proj',
});
});
it('returns null when no daemon state exists', async () => {
const readStatus = vi.fn(async () => ({
kind: 'stopped' as const,
detail: 'no state',
layout: {} as ManagedPythonDaemonLayout,
}));
const result = await tryUseManagedLocalEmbeddingsDaemon({
cliVersion: '0.5.0',
projectDir: '/work/proj',
readStatus,
});
expect(result).toBeNull();
});
it('returns null when daemon is stale', async () => {
const readStatus = vi.fn(async () => ({
kind: 'stale' as const,
detail: 'process gone',
layout: {} as ManagedPythonDaemonLayout,
}));
const result = await tryUseManagedLocalEmbeddingsDaemon({
cliVersion: '0.5.0',
projectDir: '/work/proj',
readStatus,
});
expect(result).toBeNull();
});
it('rejects daemons that do not advertise local-embeddings', async () => {
const readStatus = vi.fn(async () => ({
kind: 'running' as const,
detail: 'ok',
layout: {} as ManagedPythonDaemonLayout,
state: {
schemaVersion: 1 as const,
pid: 123,
host: '127.0.0.1' as const,
port: 4321,
version: '0.5.0',
features: ['core' as const],
startedAt: '2026-05-21T00:00:00Z',
stdoutLog: '/tmp/stdout.log',
stderrLog: '/tmp/stderr.log',
},
baseUrl: 'http://127.0.0.1:4321',
}));
const result = await tryUseManagedLocalEmbeddingsDaemon({
cliVersion: '0.5.0',
projectDir: '/work/proj',
readStatus,
});
expect(result).toBeNull();
});
});

View file

@ -1,5 +1,3 @@
import { MANAGED_SENTENCE_TRANSFORMERS_BASE_URL } from '@ktx/context';
import type { KtxProjectEmbeddingConfig } from '@ktx/context/project';
import type { KtxEmbeddingConfig } from '@ktx/llm';
import type { KtxCliIo } from './cli-runtime.js';
import {
@ -7,7 +5,12 @@ import {
type KtxManagedPythonInstallPolicy,
type ManagedPythonCommandRuntime,
} from './managed-python-command.js';
import { startManagedPythonDaemon, type ManagedPythonDaemonStartResult } from './managed-python-daemon.js';
import {
readManagedPythonDaemonStatus,
startManagedPythonDaemon,
type ManagedPythonDaemonStartResult,
type ManagedPythonDaemonStatus,
} from './managed-python-daemon.js';
export interface ManagedLocalEmbeddingsDaemon {
baseUrl: string;
@ -34,21 +37,6 @@ export interface ManagedLocalEmbeddingsOptions {
}) => Promise<ManagedPythonDaemonStartResult>;
}
export function managedLocalEmbeddingProjectConfig(input: {
model: string;
dimensions: number;
}): KtxProjectEmbeddingConfig {
return {
backend: 'sentence-transformers',
model: input.model,
dimensions: input.dimensions,
sentenceTransformers: {
base_url: MANAGED_SENTENCE_TRANSFORMERS_BASE_URL,
pathPrefix: '',
},
};
}
export function managedLocalEmbeddingHealthConfig(input: {
baseUrl: string;
model: string;
@ -93,3 +81,30 @@ export async function ensureManagedLocalEmbeddingsDaemon(
stderrLog: daemon.state.stderrLog,
};
}
export interface TryUseManagedLocalEmbeddingsOptions {
cliVersion: string;
projectDir: string;
readStatus?: typeof readManagedPythonDaemonStatus;
}
export async function tryUseManagedLocalEmbeddingsDaemon(
options: TryUseManagedLocalEmbeddingsOptions,
): Promise<ManagedLocalEmbeddingsDaemon | null> {
const readStatus = options.readStatus ?? readManagedPythonDaemonStatus;
const status: ManagedPythonDaemonStatus = await readStatus({
cliVersion: options.cliVersion,
projectDir: options.projectDir,
});
if (status.kind !== 'running') {
return null;
}
if (!status.state.features.includes('local-embeddings')) {
return null;
}
return {
baseUrl: status.baseUrl,
stdoutLog: status.state.stdoutLog,
stderrLog: status.state.stderrLog,
};
}

View file

@ -1,8 +1,10 @@
import { KtxIngestEmbeddingPortAdapter } from '@ktx/context';
import { createDefaultKtxMcpServer, createLocalProjectMcpContextPorts } from '@ktx/context/mcp';
import { createLocalProjectMemoryIngest } from '@ktx/context/memory';
import type { KtxLocalProject } from '@ktx/context/project';
import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
import type { KtxCliIo } from './cli-runtime.js';
import { resolveProjectEmbeddingProvider } from './embedding-resolution.js';
import { createKtxCliIngestQueryExecutor } from './ingest-query-executor.js';
import { createKtxCliScanConnector } from './local-scan-connectors.js';
import { createManagedPythonSemanticLayerComputePort } from './managed-python-command.js';
@ -34,10 +36,20 @@ export async function createKtxMcpServerFactory(input: {
installPolicy: 'auto',
io,
});
const resolution = await resolveProjectEmbeddingProvider(input.project, {
mode: 'use-if-running',
cliVersion: input.cliVersion,
io,
});
const embeddingService =
resolution.kind === 'configured' || resolution.kind === 'managed-running' || resolution.kind === 'managed-started'
? new KtxIngestEmbeddingPortAdapter(resolution.provider)
: null;
const contextTools = createLocalProjectMcpContextPorts(input.project, {
semanticLayerCompute,
queryExecutor,
sqlAnalysis,
embeddingService,
localScan: {
createConnector: async (connectionId) => createKtxCliScanConnector(input.project, connectionId),
},

View file

@ -1,5 +1,4 @@
import { type KtxLocalProject, type KtxProjectConnectionConfig } from '@ktx/context/project';
import { loadKtxCliProject } from './cli-project.js';
import { loadKtxProject, type KtxLocalProject, type KtxProjectConnectionConfig } from '@ktx/context/project';
import type { KtxProgressPort } from '@ktx/context/scan';
import type { KtxCliIo } from './index.js';
import type { KtxIngestArgs, KtxIngestDeps, KtxIngestProgressUpdate } from './ingest.js';
@ -869,14 +868,7 @@ export async function runKtxPublicIngest(
deps: KtxPublicIngestDeps = {},
): Promise<number> {
const loadProject =
deps.loadProject ??
((options: { projectDir: string }) =>
loadKtxCliProject({
projectDir: options.projectDir,
cliVersion: args.cliVersion ?? '0.0.0-private',
installPolicy: args.runtimeInstallPolicy ?? 'never',
io,
}));
deps.loadProject ?? ((options: { projectDir: string }) => loadKtxProject({ projectDir: options.projectDir }));
const project = await loadProject({ projectDir: args.projectDir });
if (shouldUseForegroundContextBuildView(args, io)) {
const plan = buildPublicIngestPlan(project, args);

View file

@ -1,4 +1,3 @@
import { MANAGED_SENTENCE_TRANSFORMERS_BASE_URL } from '@ktx/context';
import { buildDefaultKtxProjectConfig, type KtxProjectConfig } from '@ktx/context/project';
import { describe, expect, it } from 'vitest';
import {
@ -51,7 +50,7 @@ describe('runtime requirement detection', () => {
model: 'all-MiniLM-L6-v2',
dimensions: 384,
sentenceTransformers: {
base_url: MANAGED_SENTENCE_TRANSFORMERS_BASE_URL,
base_url: '',
},
},
},

View file

@ -1,4 +1,3 @@
import { MANAGED_SENTENCE_TRANSFORMERS_BASE_URL } from '@ktx/context';
import type {
KtxProjectConfig,
KtxProjectConnectionConfig,
@ -63,7 +62,7 @@ function requiresManagedLocalEmbeddings(embeddings: KtxProjectEmbeddingConfig):
return false;
}
const baseUrl = embeddings.sentenceTransformers?.base_url;
return baseUrl === undefined || baseUrl === '' || baseUrl === MANAGED_SENTENCE_TRANSFORMERS_BASE_URL;
return baseUrl === undefined || baseUrl === '';
}
function uniqueRequirements(requirements: KtxRuntimeRequirement[]): KtxRuntimeRequirements {

View file

@ -5,7 +5,8 @@ import {
type KtxScanWarning,
runLocalScan,
} from '@ktx/context/scan';
import { loadKtxCliProject } from './cli-project.js';
import { loadKtxProject } from '@ktx/context/project';
import { resolveProjectEmbeddingProvider } from './embedding-resolution.js';
import type { KtxCliIo } from './index.js';
import { createKtxCliLocalIngestAdapters } from './local-adapters.js';
import { createKtxCliScanConnector } from './local-scan-connectors.js';
@ -313,12 +314,15 @@ export function createCliScanProgress(
export async function runKtxScan(args: KtxScanArgs, io: KtxCliIo = process, deps: KtxScanDeps = {}): Promise<number> {
try {
const project = await loadKtxCliProject({
projectDir: args.projectDir,
cliVersion: args.cliVersion ?? '0.0.0-private',
const project = await loadKtxProject({ projectDir: args.projectDir });
const resolution = await resolveProjectEmbeddingProvider(project, {
mode: 'ensure',
installPolicy: args.runtimeInstallPolicy ?? 'never',
cliVersion: args.cliVersion ?? '0.0.0-private',
io,
});
const embeddingProvider =
resolution.kind === 'disabled' || resolution.kind === 'managed-unavailable' ? null : resolution.provider;
const managedDaemon = managedDaemonOptionsForScanRun(args, deps.runtimeIo ?? io);
const connector =
args.mode !== 'structural' || args.detectRelationships
@ -336,6 +340,7 @@ export async function runKtxScan(args: KtxScanArgs, io: KtxCliIo = process, deps
trigger: 'cli',
databaseIntrospectionUrl: args.databaseIntrospectionUrl,
connector,
embeddingProvider,
adapters: (deps.createLocalIngestAdapters ?? createKtxCliLocalIngestAdapters)(project, {
...(args.databaseIntrospectionUrl ? { databaseIntrospectionUrl: args.databaseIntrospectionUrl } : {}),
...(managedDaemon ? { managedDaemon } : {}),

View file

@ -54,9 +54,6 @@ function managedDaemon(
baseUrl,
stdoutLog: logs.stdoutLog ?? '/tmp/ktx-daemon.stdout.log',
stderrLog: logs.stderrLog ?? '/tmp/ktx-daemon.stderr.log',
env: {
KTX_MANAGED_SENTENCE_TRANSFORMERS_BASE_URL: baseUrl,
},
};
}
@ -176,8 +173,8 @@ describe('setup embeddings step', () => {
backend: 'sentence-transformers',
model: 'all-MiniLM-L6-v2',
dimensions: 384,
sentenceTransformers: { base_url: 'managed:local-embeddings', pathPrefix: '' },
});
expect(config.ingest.embeddings.sentenceTransformers).toBeUndefined();
expect(config.scan.enrichment.embeddings).toMatchObject(config.ingest.embeddings);
expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:');
expect((await readKtxSetupState(tempDir)).completed_steps).toContain('embeddings');
@ -275,8 +272,8 @@ describe('setup embeddings step', () => {
backend: 'sentence-transformers',
model: 'all-MiniLM-L6-v2',
dimensions: 384,
sentenceTransformers: { base_url: 'managed:local-embeddings', pathPrefix: '' },
});
expect(config.ingest.embeddings.sentenceTransformers).toBeUndefined();
expect(config.scan.enrichment.embeddings).toMatchObject(config.ingest.embeddings);
expect(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8')).not.toContain('completed_steps:');
expect((await readKtxSetupState(tempDir)).completed_steps).toContain('embeddings');

View file

@ -14,7 +14,6 @@ import { createStaticCliSpinner, type KtxCliSpinner } from './clack.js';
import {
ensureManagedLocalEmbeddingsDaemon,
managedLocalEmbeddingHealthConfig,
managedLocalEmbeddingProjectConfig,
type ManagedLocalEmbeddingsDaemon,
} from './managed-local-embeddings.js';
import type { KtxManagedPythonInstallPolicy } from './managed-python-command.js';
@ -455,7 +454,11 @@ export async function runKtxSetupEmbeddingsStep(
await persistEmbeddingConfig(
args.projectDir,
selectedBackend === LOCAL_EMBEDDING_BACKEND
? managedLocalEmbeddingProjectConfig({ model, dimensions })
? {
backend: 'sentence-transformers' as const,
model,
dimensions,
}
: buildProjectEmbeddingConfig({
backend: selectedBackend,
model,

View file

@ -1,7 +1,6 @@
import { mkdtemp, rm } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { MANAGED_SENTENCE_TRANSFORMERS_BASE_URL } from '@ktx/context';
import { buildDefaultKtxProjectConfig, readKtxSetupState, type KtxProjectConfig } from '@ktx/context/project';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { runKtxSetupRuntimeStep } from './setup-runtime.js';
@ -103,7 +102,6 @@ describe('runKtxSetupRuntimeStep', () => {
baseUrl: 'http://127.0.0.1:61234',
stdoutLog: join(tempDir, '.ktx', 'runtime', 'daemon.stdout.log'),
stderrLog: join(tempDir, '.ktx', 'runtime', 'daemon.stderr.log'),
env: { KTX_MANAGED_SENTENCE_TRANSFORMERS_BASE_URL: 'http://127.0.0.1:61234' },
}));
const config: KtxProjectConfig = {
...buildDefaultKtxProjectConfig(),
@ -113,7 +111,7 @@ describe('runKtxSetupRuntimeStep', () => {
backend: 'sentence-transformers',
model: 'all-MiniLM-L6-v2',
dimensions: 384,
sentenceTransformers: { base_url: MANAGED_SENTENCE_TRANSFORMERS_BASE_URL },
sentenceTransformers: { base_url: '' },
},
},
};

View file

@ -77,12 +77,24 @@ describe('runKtxSl', () => {
expect(validateIo.stdout()).toContain('Valid semantic-layer source: warehouse/orders');
const listIo = makeIo();
await expect(runKtxSl({ command: 'list', projectDir, connectionId: 'warehouse' }, listIo.io)).resolves.toBe(0);
await expect(
runKtxSl({ command: 'list', projectDir, connectionId: 'warehouse', cliVersion: '0.0.0-test' }, listIo.io),
).resolves.toBe(0);
expect(listIo.stdout()).toContain('warehouse\torders\tcolumns=1\tmeasures=0\tjoins=0');
const searchIo = makeIo();
await expect(
runKtxSl({ command: 'search', projectDir, connectionId: 'warehouse', query: 'order', json: true }, searchIo.io),
runKtxSl(
{
command: 'search',
projectDir,
connectionId: 'warehouse',
query: 'order',
json: true,
cliVersion: '0.0.0-test',
},
searchIo.io,
),
).resolves.toBe(0);
expect(JSON.parse(searchIo.stdout())).toMatchObject({
kind: 'list',
@ -106,7 +118,14 @@ describe('runKtxSl', () => {
const searchIo = makeIo();
await expect(
runKtxSl(
{ command: 'search', projectDir, connectionId: 'warehouse', query: 'order', output: 'pretty' },
{
command: 'search',
projectDir,
connectionId: 'warehouse',
query: 'order',
output: 'pretty',
cliVersion: '0.0.0-test',
},
searchIo.io,
),
).resolves.toBe(0);
@ -136,7 +155,14 @@ describe('runKtxSl', () => {
const listIo = makeIo();
await expect(
runKtxSl(
{ command: 'search', projectDir, connectionId: 'warehouse', query: 'paid', json: true },
{
command: 'search',
projectDir,
connectionId: 'warehouse',
query: 'paid',
json: true,
cliVersion: '0.0.0-test',
},
listIo.io,
),
).resolves.toBe(0);
@ -575,7 +601,7 @@ joins: []
const listIo = makeIo();
const code = await runKtxSl(
{ command: 'list', projectDir, connectionId: 'warehouse', output: 'json' },
{ command: 'list', projectDir, connectionId: 'warehouse', output: 'json', cliVersion: '0.0.0-test' },
listIo.io,
);
expect(code).toBe(0);
@ -601,13 +627,80 @@ joins: []
});
});
it('search prints embeddings status when results are empty', async () => {
const stderr: string[] = [];
const io = {
stdout: { write: (_chunk: string) => {} },
stderr: {
write: (chunk: string) => {
stderr.push(chunk);
},
},
};
const projectDir = join(tempDir, 'empty-status');
const project = await initKtxProject({ projectDir });
await expect(
runKtxSl(
{
command: 'search',
projectDir: project.projectDir,
query: 'nope',
cliVersion: '0.5.0',
},
io,
{
loadProject: async () => project,
resolveEmbeddingProvider: async () => ({
kind: 'managed-unavailable',
reason: 'managed embeddings daemon is not running',
}),
searchLocalSlSources: async () => [],
},
),
).resolves.toBe(0);
expect(stderr.join('')).toMatch(/embeddings: unavailable/);
expect(stderr.join('')).toMatch(/managed embeddings daemon is not running/);
});
it('passes a managed-daemon-backed embedding service into the search', async () => {
const projectDir = join(tempDir, 'resolver-project');
const project = await initKtxProject({ projectDir });
const search = vi.fn(async () => []);
const searchIo = makeIo();
await expect(
runKtxSl(
{
command: 'search',
projectDir: project.projectDir,
query: 'income',
cliVersion: '0.5.0',
json: true,
},
searchIo.io,
{
loadProject: async () => project,
resolveEmbeddingProvider: async () => ({
kind: 'managed-running',
provider: { id: 'fake' } as never,
baseUrl: 'http://127.0.0.1:51234',
}),
searchLocalSlSources: search,
},
),
).resolves.toBe(0);
expect(search).toHaveBeenCalledWith(
project,
expect.objectContaining({ embeddingService: expect.any(Object) }),
);
});
it('emits sl list with grouping and Clack-style framing when output=pretty', async () => {
const projectDir = join(tempDir, 'project');
await seedSlSource({ projectDir });
const listIo = makeIo();
const code = await runKtxSl(
{ command: 'list', projectDir, connectionId: 'warehouse', output: 'pretty' },
{ command: 'list', projectDir, connectionId: 'warehouse', output: 'pretty', cliVersion: '0.0.0-test' },
listIo.io,
);
expect(code).toBe(0);

View file

@ -1,22 +1,22 @@
import { readFile } from 'node:fs/promises';
import { createDefaultLocalQueryExecutor, type KtxSqlQueryExecutorPort } from '@ktx/context/connections';
import {
createLocalKtxEmbeddingProviderFromConfig,
KtxIngestEmbeddingPortAdapter,
type KtxEmbeddingPort,
} from '@ktx/context';
import { KtxIngestEmbeddingPortAdapter, type KtxEmbeddingPort } from '@ktx/context';
import type { KtxSemanticLayerComputePort } from '@ktx/context/daemon';
import { loadKtxProject, type KtxLocalProject } from '@ktx/context/project';
import {
compileLocalSlQuery,
listLocalSlSources,
readLocalSlSource,
searchLocalSlSources,
searchLocalSlSources as defaultSearchLocalSlSources,
validateLocalSlSource,
type LocalSlSourceSearchResult,
type LocalSlSourceSummary,
type SemanticLayerQueryInput,
} from '@ktx/context/sl';
import {
resolveProjectEmbeddingProvider,
type EmbeddingProviderResolution,
} from './embedding-resolution.js';
import type { PrintListColumn } from './io/print-list.js';
import {
createManagedPythonSemanticLayerComputePort,
@ -29,7 +29,14 @@ profileMark('module:sl');
type SlQueryFormat = 'json' | 'sql';
export type KtxSlArgs =
| { command: 'list'; projectDir: string; connectionId?: string; output?: string; json?: boolean }
| {
command: 'list';
projectDir: string;
connectionId?: string;
output?: string;
json?: boolean;
cliVersion: string;
}
| {
command: 'search';
projectDir: string;
@ -38,6 +45,7 @@ export type KtxSlArgs =
limit?: number;
output?: string;
json?: boolean;
cliVersion: string;
}
| { command: 'validate'; projectDir: string; connectionId: string; sourceName: string }
| {
@ -60,8 +68,8 @@ interface KtxSlIo {
interface KtxSlDeps {
loadProject?: typeof loadKtxProject;
embeddingService?: KtxEmbeddingPort | null;
createEmbeddingProvider?: typeof createLocalKtxEmbeddingProviderFromConfig;
resolveEmbeddingProvider?: typeof resolveProjectEmbeddingProvider;
searchLocalSlSources?: typeof defaultSearchLocalSlSources;
createSemanticLayerCompute?: () => KtxSemanticLayerComputePort;
createManagedSemanticLayerCompute?: (options: {
cliVersion: string;
@ -71,14 +79,15 @@ interface KtxSlDeps {
createQueryExecutor?: () => KtxSqlQueryExecutorPort;
}
function slSearchEmbeddingService(project: KtxLocalProject, deps: KtxSlDeps): KtxEmbeddingPort | null {
if ('embeddingService' in deps) {
return deps.embeddingService ?? null;
function resolutionToEmbeddingPort(resolution: EmbeddingProviderResolution): KtxEmbeddingPort | null {
if (
resolution.kind === 'configured' ||
resolution.kind === 'managed-running' ||
resolution.kind === 'managed-started'
) {
return new KtxIngestEmbeddingPortAdapter(resolution.provider);
}
const provider = (deps.createEmbeddingProvider ?? createLocalKtxEmbeddingProviderFromConfig)(
project.config.ingest.embeddings,
);
return provider ? new KtxIngestEmbeddingPortAdapter(provider) : null;
return null;
}
async function printSlSources(input: {
@ -188,12 +197,24 @@ export async function runKtxSl(args: KtxSlArgs, io: KtxSlIo = process, deps: Ktx
return 0;
}
if (args.command === 'search') {
const sources = await searchLocalSlSources(project, {
const resolver = deps.resolveEmbeddingProvider ?? resolveProjectEmbeddingProvider;
const resolution = await resolver(project, {
mode: 'use-if-running',
cliVersion: args.cliVersion,
io,
});
const embeddingService = resolutionToEmbeddingPort(resolution);
const search = deps.searchLocalSlSources ?? defaultSearchLocalSlSources;
const sources = await search(project, {
connectionId: args.connectionId,
query: args.query,
embeddingService: slSearchEmbeddingService(project, deps),
embeddingService,
limit: args.limit,
});
if (sources.length === 0 && resolution.kind === 'managed-unavailable' && !args.json) {
const { SYMBOLS } = await import('./io/symbols.js');
io.stderr.write(`embeddings: unavailable ${SYMBOLS.emDash} ${resolution.reason}\n`);
}
await printSlSources({
rows: sources,
emptyMessage: `No semantic-layer sources matched "${args.query}" in ${project.projectDir}`,