apunkt/ktx
1
0
Fork 0
mirror of https://github.com/Kaelio/ktx.git synced 2026-06-28 08:49:38 +02:00
Code Issues Projects Releases Packages Wiki Activity

feat: npm-managed Python runtime for @kaelio/ktx (#7)

Browse source 
* docs: add npm managed python runtime design

* build: add bundled python runtime wheel builder

* build: make local embedding dependencies optional

* build: bundle python runtime wheel in cli artifacts

* build: track bundled python runtime release artifact

* test: verify bundled python runtime wheel

* docs: add plan for bundled python runtime wheel

* test: cover managed python runtime lifecycle

* feat: add managed python runtime installer

* feat: add runtime command runner

* feat: expose runtime management commands

* test: verify managed python runtime commands

* docs: add plan for managed python runtime installer

* feat: add managed python command helper

* feat: use managed runtime for sl query compute

* feat: route sl query managed runtime policy

* docs: add plan for managed runtime sl query integration

* feat: add managed runtime daemon metadata

* feat: manage python daemon lifecycle

* feat: add runtime daemon start stop commands

* fix: verify managed runtime daemon lifecycle

* docs: add plan for managed runtime daemon lifecycle

* feat: add managed local embeddings config marker

* feat: add managed local embeddings daemon helper

* feat: use managed runtime for local embedding setup

* feat: pass managed runtime policy through setup

* docs: add plan for managed local embeddings runtime

* feat: read CLI package metadata dynamically

* feat: assemble public kaelio ktx npm package

* feat: release one public kaelio ktx npm artifact

* test: cover public kaelio ktx package invocations

* chore: verify public kaelio ktx package artifacts

* docs: add plan for public kaelio ktx npm package

* test: verify managed runtime in public package smoke

* test: finalize managed runtime release smoke

* docs: add plan for managed runtime release smoke

* test: specify local embeddings release smoke

* feat: add local embeddings runtime smoke

* chore: register local embeddings smoke

* fix: verify local embeddings smoke

* fix: restore artifact smoke python env helper

* docs: add plan for managed local embeddings release smoke

* refactor: share managed runtime install policy parsing

* feat: use managed runtime for agent semantic queries

* feat: use managed runtime for MCP semantic compute

* docs: add plan for managed agent and MCP semantic runtime

* feat(cli): add managed daemon HTTP helpers

* feat(cli): route local adapters through managed daemon

* feat(cli): use managed daemon for ingest helpers

* feat(cli): pass managed daemon options to scan

* feat(context): pass MCP ingest pull config options

* feat(cli): pass managed daemon options to serve ingest

* test: verify managed local ingest daemon runtime

* docs: add plan for managed local ingest daemon runtime

* docs: align managed runtime examples

* docs: add plan for managed runtime docs cleanup

* test: cover published package runtime smoke commands

* test: validate published package smoke outputs

* docs: add plan for published package runtime smoke

* build: stamp public npm package version

* release: add npm public release policy

* release: add guarded npm publish script

* release: document public npm release handoff

* docs: add plan for public npm release handoff

* test: cover managed runtime prune in package smoke

* docs: document managed runtime prune

* docs: add plan for managed runtime prune smoke and docs

* chore: encode uv runtime prerequisite policy

* fix: clarify missing uv runtime error

* docs: document uv runtime prerequisite

* docs: add plan for uv runtime prerequisite contract

* refactor: limit release artifacts to public package runtime

* chore: align release policy with bundled runtime wheel

* docs: describe single public runtime artifact surface

* test: verify single public runtime artifact contract

* docs: add plan for single public runtime artifact cleanup

* fix: align local embeddings smoke with public version

* docs: add plan for local embeddings smoke public version

* release: soft-launch as @kaelio/ktx@0.1.0-rc.0 on next tag

Publish target moves to the pre-release version 0.1.0-rc.0 under the next
dist-tag so npm install @kaelio/ktx (which resolves to latest) does not
pick up the soft-launch build. Users opt in via @kaelio/ktx@next.

* Fix release script boundary checks

* Remove PostHog from public package bundle
This commit is contained in:
Andrey Avtomonov 2026-05-11 15:50:34 +02:00 • committed by GitHub
parent 075764fe77
commit 9dad936ac7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
99 changed files with 25375 additions and 1538 deletions
Download patch file Download diff file Expand all files Collapse all files

11
packages/context/src/ingest/local-ingest.ts
View file

@ -41,10 +41,17 @@ export interface RunLocalIngestOptions {
export interface LocalIngestMcpOptions
extends Pick<
RunLocalIngestOptions,
'agentRunner' | 'llmProvider' | 'memoryModel' | 'semanticLayerCompute' | 'queryExecutor' | 'logger'
> {
| 'agentRunner'
| 'llmProvider'
| 'memoryModel'
| 'semanticLayerCompute'
| 'queryExecutor'
| 'logger'
| 'pullConfigOptions'
> {
adapters?: SourceAdapter[];
jobIdFactory?: () => string;
runLocalIngest?: (options: RunLocalIngestOptions) => Promise<LocalIngestResult>;
runLocalMetabaseIngest?: (options: RunLocalMetabaseIngestOptions) => Promise<LocalMetabaseFanoutResult>;
}

2
packages/context/src/llm/index.ts
View file

@ -11,6 +11,8 @@ export {
summarizeKtxLlmDebugRequest,
} from './debug-request-recorder.js';
export {
MANAGED_SENTENCE_TRANSFORMERS_BASE_URL,
MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV,
createLocalKtxEmbeddingProviderFromConfig,
createLocalKtxLlmProviderFromConfig,
resolveLocalKtxEmbeddingConfig,

41
packages/context/src/llm/local-config.test.ts
View file

@ -5,6 +5,8 @@ import {
type KtxProjectLlmConfig,
} from '../project/config.js';
import {
MANAGED_SENTENCE_TRANSFORMERS_BASE_URL,
MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV,
createLocalKtxEmbeddingProviderFromConfig,
createLocalKtxLlmProviderFromConfig,
resolveLocalKtxEmbeddingConfig,
@ -104,6 +106,45 @@ describe('local KTX embedding config', () => {
});
});
it('resolves managed sentence-transformers config from the CLI-provided daemon URL', () => {
const config: KtxProjectEmbeddingConfig = {
backend: 'sentence-transformers',
model: 'all-MiniLM-L6-v2',
dimensions: 384,
sentenceTransformers: {
base_url: MANAGED_SENTENCE_TRANSFORMERS_BASE_URL,
pathPrefix: '',
},
batchSize: 32,
};
expect(
resolveLocalKtxEmbeddingConfig(config, {
[MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV]: 'http://127.0.0.1:61234',
}),
).toEqual({
backend: 'sentence-transformers',
model: 'all-MiniLM-L6-v2',
dimensions: 384,
sentenceTransformers: { baseURL: 'http://127.0.0.1:61234', pathPrefix: '' },
batchSize: 32,
});
});
it('returns null for managed sentence-transformers when no daemon URL is available', () => {
const config: KtxProjectEmbeddingConfig = {
backend: 'sentence-transformers',
model: 'all-MiniLM-L6-v2',
dimensions: 384,
sentenceTransformers: {
base_url: MANAGED_SENTENCE_TRANSFORMERS_BASE_URL,
pathPrefix: '',
},
};
expect(resolveLocalKtxEmbeddingConfig(config, {})).toBeNull();
});
it('constructs deterministic embeddings from the default project config', () => {
const createKtxEmbeddingProvider = vi.fn(() => ({}) as never);
const provider = createLocalKtxEmbeddingProviderFromConfig(

32
packages/context/src/llm/local-config.ts
View file

@ -16,6 +16,9 @@ interface LocalConfigDeps {
createKtxEmbeddingProvider?: typeof createKtxEmbeddingProvider;
}
export const MANAGED_SENTENCE_TRANSFORMERS_BASE_URL = 'managed:local-embeddings';
export const MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV = 'KTX_MANAGED_SENTENCE_TRANSFORMERS_BASE_URL';
function resolveOptional(value: string | undefined, env: NodeJS.ProcessEnv): string | undefined {
return resolveKtxConfigReference(value, env) || undefined;
}
@ -89,6 +92,19 @@ export function createLocalKtxLlmProviderFromConfig(
return resolved ? (deps.createKtxLlmProvider ?? createKtxLlmProvider)(resolved) : null;
}
function resolveSentenceTransformersBaseUrl(
value: string | undefined,
env: NodeJS.ProcessEnv,
): string | undefined {
if (!value) {
return undefined;
}
if (value === MANAGED_SENTENCE_TRANSFORMERS_BASE_URL) {
return resolveOptional(`env:${MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV}`, env);
}
return value;
}
export function resolveLocalKtxEmbeddingConfig(
config: KtxProjectEmbeddingConfig,
env: NodeJS.ProcessEnv,
@ -96,6 +112,22 @@ export function resolveLocalKtxEmbeddingConfig(
if (config.backend === 'none') {
return null;
}
if (config.backend === 'sentence-transformers') {
const baseURL = resolveSentenceTransformersBaseUrl(config.sentenceTransformers?.base_url, env);
if (!baseURL) {
return null;
}
return {
backend: config.backend,
model: config.model ?? 'all-MiniLM-L6-v2',
dimensions: config.dimensions,
sentenceTransformers: {
baseURL,
pathPrefix: config.sentenceTransformers?.pathPrefix,
},
batchSize: config.batchSize,
};
}
return {
backend: config.backend,
model: config.model ?? 'deterministic',

59
packages/context/src/mcp/local-project-ports.test.ts
View file

@ -845,6 +845,65 @@ describe('createLocalProjectMcpContextPorts', () => {
expect(agentRunner.runLoop).toHaveBeenCalledTimes(1);
});
it('passes local ingest pull-config options into runLocalIngest', async () => {
const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' });
project.config.connections.warehouse = { driver: 'postgres' };
project.config.ingest.adapters = ['looker'];
const runLocalIngest = vi.fn(async () => ({
result: { ok: true },
report: {
id: 'report-1',
runId: 'run-1',
jobId: 'job-1',
sourceKey: 'looker',
connectionId: 'warehouse',
body: {
syncId: 'sync-1',
workUnits: [],
failedWorkUnits: [],
diffSummary: { added: 0, modified: 0, deleted: 0, unchanged: 0 },
provenanceRows: [],
},
},
}) as never);
const ports = createLocalProjectMcpContextPorts(project, {
localIngest: {
adapters: [
{ source: 'looker', skillNames: [], detect: async () => true, chunk: async () => ({ workUnits: [] }) },
],
pullConfigOptions: {
looker: {
daemonBaseUrl: 'http://127.0.0.1:61234',
},
},
runLocalIngest,
},
});
await expect(
ports.ingest?.trigger({
adapter: 'looker',
connectionId: 'warehouse',
trigger: 'manual_resync',
config: {},
}),
).resolves.toMatchObject({
runId: 'run-1',
jobId: 'job-1',
reportId: 'report-1',
});
expect(runLocalIngest).toHaveBeenCalledWith(
expect.objectContaining({
pullConfigOptions: {
looker: {
daemonBaseUrl: 'http://127.0.0.1:61234',
},
},
}),
);
});
it('triggers fetch-capable local ingest without sourceDir config', async () => {
const project = await initKtxProject({ projectDir: tempDir, projectName: 'warehouse' });
project.config.connections.warehouse = {

5
packages/context/src/mcp/local-project-ports.ts
View file

@ -586,6 +586,7 @@ export function createLocalProjectMcpContextPorts(
metabaseConnectionId: input.connectionId,
trigger: input.trigger,
jobIdFactory: options.localIngest?.jobIdFactory,
pullConfigOptions: options.localIngest?.pullConfigOptions,
agentRunner: options.localIngest?.agentRunner,
llmProvider: options.localIngest?.llmProvider,
memoryModel: options.localIngest?.memoryModel,
@ -610,12 +611,14 @@ export function createLocalProjectMcpContextPorts(
};
}
const result = await runLocalIngest({
const executeLocalIngest = options.localIngest?.runLocalIngest ?? runLocalIngest;
const result = await executeLocalIngest({
project,
adapters: options.localIngest?.adapters ?? createDefaultLocalIngestAdapters(project),
adapter: input.adapter,
connectionId: input.connectionId,
sourceDir,
pullConfigOptions: options.localIngest?.pullConfigOptions,
trigger: input.trigger,
jobId: options.localIngest?.jobIdFactory?.(),
agentRunner: options.localIngest?.agentRunner,

4
packages/context/src/package-exports.test.ts
View file

@ -132,6 +132,10 @@ describe('@ktx/context package exports', () => {
expect(root.assertSearchBackendConformanceCase).toBeTypeOf('function');
expect(root.assertSearchBackendCapabilities).toBeTypeOf('function');
expect(root.createLocalKtxEmbeddingProviderFromConfig).toBeTypeOf('function');
expect(root.MANAGED_SENTENCE_TRANSFORMERS_BASE_URL).toBe('managed:local-embeddings');
expect(root.MANAGED_SENTENCE_TRANSFORMERS_BASE_URL_ENV).toBe(
'KTX_MANAGED_SENTENCE_TRANSFORMERS_BASE_URL',
);
expect(agent).toBeDefined();
expect(agent.AgentRunnerService).toBeTypeOf('function');
expect(root.AgentRunnerService).toBeTypeOf('function');