mirror of
https://github.com/Kaelio/ktx.git
synced 2026-06-22 08:38:08 +02:00
test: split cli tests from source tree (#216)
* feat(cli): define full warehouse dialect contract
* test(cli): keep dialect edge tests focused
* fix(cli): stabilize dialect contract foundation
* refactor(connectors): own read-only query preparation
* refactor(connectors): resolve dialects through registry
* refactor(connectors): keep concrete dialect classes internal
* chore(workspace): enforce dialect import boundary
* refactor(cli): resolve relationship dialect at scan boundary
* refactor(cli): use dialect display parsing for entity details
* refactor(cli): use dialect display parsing for warehouse catalog
* refactor(cli): use dialect SQL in relationship workflows
* test(cli): verify solid dialect scan workflow closure
* test: split cli tests from source tree
* refactor(cli): standardize BigQuery scope listing
* feat(sqlite): implement connector scope listing
* test(connectors): cover required table listing
* feat(cli): add warehouse driver registry
* refactor(setup): route scope discovery through driver registry
* refactor(cli): route local query execution through driver registry
* refactor(historic-sql): route dialect support through driver registry
* refactor(cli): test warehouse connections through driver registry
* fix(cli): close driver registry type export gaps
* Improve setup daemon diagnostics
* refactor(setup): centralize rail-prefixed diagnostics + query-history fallback
Extract errorMessage, writePrefixedLines, and flushPrefixedBufferedCommandOutput
into clack.ts so the setup wizard, managed daemons, and embedding/agent steps
share one rail-formatted writer. setup-databases.ts also adds a
"disable query history and retry" option when the schema-context build fails
and query history is the likely culprit, surfaced via a new
failed-query-history-unavailable status.
* fix(cli): carry catalog through the picker so BigQuery/Snowflake/SQL Server scope filters match
The setup picker's KtxTableListEntry was a 2-level { schema, name }, so
qualifiedTableId always wrote db.name into enabled_tables. When BigQuery,
Snowflake, or SQL Server later ran fast ingest, their introspect step filtered
the scope set with scopedTableNames(scope, { catalog: projectId|database, db })
— catalog was non-null on the introspect side but null in the scope refs, so
every entry was rejected, the live-database adapter staged zero table files,
and detect() failed with 'Adapter "live-database" did not recognize fetched
source output'.
Align the picker boundary with the canonical 3-level KtxTableRef:
- Add catalog: string | null to KtxTableListEntry.
- BigQuery/Snowflake/SQL Server listTables populate catalog from the
resolved projectId / database; Postgres/MySQL/ClickHouse/SQLite set null.
- qualifiedTableId emits catalog.schema.name when catalog is non-null
(resolveEnabledTables already accepts the 3-part shape) and
schemasFromEnabledTables now goes through parseDottedTableEntry so it
recovers the schema correctly from both 2-part and 3-part entries.
- Export parseDottedTableEntry from enabled-tables.ts (@internal) for picker
reuse.
Update listTables expectations in all seven connector tests and the setup /
picker test fixtures. Add a picker regression test that covers the
catalog-bearing round-trip (save + refine).
* fix(cli): allow debug telemetry under opt-out env
This commit is contained in:
Andrey Avtomonov
GitHub
parent
924868841d
commit
56985b7e09
548 changed files with 5048 additions and 2228 deletions
|
|
@ -1,337 +0,0 @@
|
|||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
vi.mock('ai', () => ({
|
||||
generateText: vi.fn(),
|
||||
stepCountIs: (n: number) => n,
|
||||
tool: (def: unknown) => def,
|
||||
}));
|
||||
|
||||
import { generateText } from 'ai';
|
||||
import { AiSdkKtxLlmRuntime } from './ai-sdk-runtime.js';
|
||||
import type { RunLoopStepInfo } from './runtime-port.js';
|
||||
|
||||
describe('AiSdkKtxLlmRuntime.runAgentLoop', () => {
|
||||
let runtime: AiSdkKtxLlmRuntime;
|
||||
const llmProvider = {
|
||||
getModel: vi.fn().mockReturnValue({ modelId: 'claude-sonnet-4-6', provider: 'anthropic' }),
|
||||
getModelByName: vi.fn(),
|
||||
cacheMarker: vi.fn(),
|
||||
repairToolCallHandler: vi.fn(),
|
||||
thinkingProviderOptions: vi.fn(),
|
||||
telemetryConfig: vi.fn(),
|
||||
promptCachingConfig: vi.fn(() => ({
|
||||
enabled: false,
|
||||
systemTtl: '1h',
|
||||
toolsTtl: '1h',
|
||||
historyTtl: '5m',
|
||||
cacheSystem: true,
|
||||
cacheTools: true,
|
||||
cacheHistory: true,
|
||||
vertexFallbackTo5m: false,
|
||||
})),
|
||||
activeBackend: vi.fn(() => 'anthropic'),
|
||||
};
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
runtime = new AiSdkKtxLlmRuntime({ llmProvider: llmProvider as any });
|
||||
});
|
||||
|
||||
afterEach(() => vi.clearAllMocks());
|
||||
|
||||
it('passes systemPrompt, userPrompt, tools, and step budget through to generateText', async () => {
|
||||
(generateText as any).mockResolvedValue({ text: 'ok', toolCalls: [], steps: [] });
|
||||
const repairHandler = vi.fn();
|
||||
llmProvider.repairToolCallHandler.mockReturnValueOnce(repairHandler);
|
||||
const tools = { noop: { description: 'noop', inputSchema: {}, execute: vi.fn() } };
|
||||
await runtime.runAgentLoop({
|
||||
modelRole: 'candidateExtraction',
|
||||
systemPrompt: 'SYS',
|
||||
userPrompt: 'USR',
|
||||
toolSet: tools as any,
|
||||
stepBudget: 17,
|
||||
telemetryTags: { source: 'test' },
|
||||
});
|
||||
const call = (generateText as any).mock.calls[0][0];
|
||||
expect(call.system).toEqual({ role: 'system', content: 'SYS' });
|
||||
expect(call.messages).toEqual([{ role: 'user', content: 'USR' }]);
|
||||
expect(call.prompt).toBeUndefined();
|
||||
expect(call.tools.noop).toEqual(
|
||||
expect.objectContaining({
|
||||
description: 'noop',
|
||||
inputSchema: {},
|
||||
execute: expect.any(Function),
|
||||
toModelOutput: expect.any(Function),
|
||||
}),
|
||||
);
|
||||
expect(call.stopWhen).toBe(17);
|
||||
expect(call.temperature).toBe(0);
|
||||
expect(call.experimental_repairToolCall).toBe(repairHandler);
|
||||
expect(llmProvider.getModel).toHaveBeenCalledWith('candidateExtraction');
|
||||
expect(llmProvider.repairToolCallHandler).toHaveBeenCalledWith({ source: 'ktx-agent-runner' });
|
||||
});
|
||||
|
||||
it('returns stopReason=natural when the loop completes without error', async () => {
|
||||
(generateText as any).mockResolvedValue({ text: 'done', toolCalls: [], steps: [] });
|
||||
const result = await runtime.runAgentLoop({
|
||||
modelRole: 'candidateExtraction',
|
||||
systemPrompt: 'system',
|
||||
userPrompt: 'user',
|
||||
toolSet: {},
|
||||
stepBudget: 10,
|
||||
telemetryTags: {},
|
||||
});
|
||||
expect(result.stopReason).toBe('natural');
|
||||
expect(result.error).toBeUndefined();
|
||||
expect(llmProvider.getModel).toHaveBeenCalledWith('candidateExtraction');
|
||||
expect(generateText).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
system: { role: 'system', content: 'system' },
|
||||
messages: [{ role: 'user', content: 'user' }],
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('returns stopReason=error with the error on generateText failure', async () => {
|
||||
const err = new Error('LLM unavailable');
|
||||
(generateText as any).mockRejectedValue(err);
|
||||
const result = await runtime.runAgentLoop({
|
||||
modelRole: 'candidateExtraction',
|
||||
systemPrompt: '',
|
||||
userPrompt: '',
|
||||
toolSet: {},
|
||||
stepBudget: 10,
|
||||
telemetryTags: {},
|
||||
});
|
||||
expect(result.stopReason).toBe('error');
|
||||
expect(result.error).toBe(err);
|
||||
});
|
||||
|
||||
it('invokes caller onStepFinish with incrementing stepIndex and total budget', async () => {
|
||||
const calls: RunLoopStepInfo[] = [];
|
||||
(generateText as any).mockImplementation(async (opts: any) => {
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await opts.onStepFinish({});
|
||||
}
|
||||
return { text: 'ok', toolCalls: [], steps: [] };
|
||||
});
|
||||
|
||||
await runtime.runAgentLoop({
|
||||
modelRole: 'candidateExtraction',
|
||||
systemPrompt: '',
|
||||
userPrompt: '',
|
||||
toolSet: {},
|
||||
stepBudget: 10,
|
||||
telemetryTags: {},
|
||||
onStepFinish: (info) => {
|
||||
calls.push(info);
|
||||
},
|
||||
});
|
||||
|
||||
expect(calls).toEqual([
|
||||
{ stepIndex: 1, stepBudget: 10 },
|
||||
{ stepIndex: 2, stepBudget: 10 },
|
||||
{ stepIndex: 3, stepBudget: 10 },
|
||||
]);
|
||||
});
|
||||
|
||||
it('swallows errors thrown from caller onStepFinish without aborting the loop', async () => {
|
||||
(generateText as any).mockImplementation(async (opts: any) => {
|
||||
await opts.onStepFinish({});
|
||||
return { text: 'ok', toolCalls: [], steps: [] };
|
||||
});
|
||||
|
||||
const result = await runtime.runAgentLoop({
|
||||
modelRole: 'candidateExtraction',
|
||||
systemPrompt: '',
|
||||
userPrompt: '',
|
||||
toolSet: {},
|
||||
stepBudget: 10,
|
||||
telemetryTags: {},
|
||||
onStepFinish: () => {
|
||||
throw new Error('boom');
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.stopReason).toBe('natural');
|
||||
});
|
||||
|
||||
it('forwards telemetryTags.source through experimental_telemetry metadata', async () => {
|
||||
(generateText as any).mockResolvedValue({ text: 'ok', toolCalls: [], steps: [] });
|
||||
const telemetryConfigEnabled = {
|
||||
isEnabled: () => true,
|
||||
devtoolsEnabled: false,
|
||||
appSettingsService: {
|
||||
settings: { telemetry: { recordInputs: false, recordOutputs: false } },
|
||||
},
|
||||
systemConfigService: {
|
||||
config: { instance: { name: 'test-instance' } },
|
||||
},
|
||||
} as any;
|
||||
const runtimeWithTelemetry = new AiSdkKtxLlmRuntime({
|
||||
llmProvider: llmProvider as any,
|
||||
telemetry: {
|
||||
createTelemetry: (tags) => ({
|
||||
isEnabled: telemetryConfigEnabled.isEnabled(),
|
||||
metadata: {
|
||||
source: tags.source ?? 'RESEARCH',
|
||||
jobId: tags.jobId,
|
||||
unitKey: tags.unitKey,
|
||||
},
|
||||
}),
|
||||
},
|
||||
});
|
||||
await runtimeWithTelemetry.runAgentLoop({
|
||||
modelRole: 'candidateExtraction',
|
||||
systemPrompt: '',
|
||||
userPrompt: '',
|
||||
toolSet: {},
|
||||
stepBudget: 10,
|
||||
telemetryTags: { source: 'metabase', jobId: 'job-123', unitKey: 'u/1' },
|
||||
});
|
||||
const call = (generateText as any).mock.calls[0][0];
|
||||
expect(call.experimental_telemetry.metadata.source).toBe('metabase');
|
||||
});
|
||||
|
||||
it('defaults to source=RESEARCH when telemetryTags omits source', async () => {
|
||||
(generateText as any).mockResolvedValue({ text: 'ok', toolCalls: [], steps: [] });
|
||||
const telemetryConfigEnabled = {
|
||||
isEnabled: () => true,
|
||||
devtoolsEnabled: false,
|
||||
appSettingsService: {
|
||||
settings: { telemetry: { recordInputs: false, recordOutputs: false } },
|
||||
},
|
||||
systemConfigService: {
|
||||
config: { instance: { name: 'test-instance' } },
|
||||
},
|
||||
} as any;
|
||||
const runtimeWithTelemetry = new AiSdkKtxLlmRuntime({
|
||||
llmProvider: llmProvider as any,
|
||||
telemetry: {
|
||||
createTelemetry: (tags) => ({
|
||||
isEnabled: telemetryConfigEnabled.isEnabled(),
|
||||
metadata: {
|
||||
source: tags.source ?? 'RESEARCH',
|
||||
jobId: tags.jobId,
|
||||
unitKey: tags.unitKey,
|
||||
},
|
||||
}),
|
||||
},
|
||||
});
|
||||
await runtimeWithTelemetry.runAgentLoop({
|
||||
modelRole: 'candidateExtraction',
|
||||
systemPrompt: '',
|
||||
userPrompt: '',
|
||||
toolSet: {},
|
||||
stepBudget: 10,
|
||||
telemetryTags: { operationName: 'memory-agent-ingest' },
|
||||
});
|
||||
const call = (generateText as any).mock.calls[0][0];
|
||||
expect(call.experimental_telemetry.metadata.source).toBe('RESEARCH');
|
||||
});
|
||||
|
||||
it('forwards jobId and unitKey through experimental_telemetry metadata', async () => {
|
||||
(generateText as any).mockResolvedValue({ text: 'ok', toolCalls: [], steps: [] });
|
||||
const telemetryConfigEnabled = {
|
||||
isEnabled: () => true,
|
||||
devtoolsEnabled: false,
|
||||
appSettingsService: {
|
||||
settings: { telemetry: { recordInputs: false, recordOutputs: false } },
|
||||
},
|
||||
systemConfigService: {
|
||||
config: { instance: { name: 'test-instance' } },
|
||||
},
|
||||
} as any;
|
||||
const runtimeWithTelemetry = new AiSdkKtxLlmRuntime({
|
||||
llmProvider: llmProvider as any,
|
||||
telemetry: {
|
||||
createTelemetry: (tags) => ({
|
||||
isEnabled: telemetryConfigEnabled.isEnabled(),
|
||||
metadata: {
|
||||
source: tags.source ?? 'RESEARCH',
|
||||
jobId: tags.jobId,
|
||||
unitKey: tags.unitKey,
|
||||
},
|
||||
}),
|
||||
},
|
||||
});
|
||||
await runtimeWithTelemetry.runAgentLoop({
|
||||
modelRole: 'candidateExtraction',
|
||||
systemPrompt: '',
|
||||
userPrompt: '',
|
||||
toolSet: {},
|
||||
stepBudget: 10,
|
||||
telemetryTags: { source: 'metabase', jobId: 'job-777', unitKey: 'sources/users' },
|
||||
});
|
||||
const call = (generateText as any).mock.calls[0][0];
|
||||
expect(call.experimental_telemetry.metadata.jobId).toBe('job-777');
|
||||
expect(call.experimental_telemetry.metadata.unitKey).toBe('sources/users');
|
||||
});
|
||||
|
||||
it('records a sanitized LLM debug request when a recorder is injected', async () => {
|
||||
(generateText as any).mockResolvedValue({ text: 'ok', toolCalls: [], steps: [] });
|
||||
const record = vi.fn();
|
||||
const provider = {
|
||||
...llmProvider,
|
||||
cacheMarker: vi.fn((ttl: '5m' | '1h') => ({
|
||||
anthropic: { cacheControl: { type: 'ephemeral' as const, ttl } },
|
||||
})),
|
||||
promptCachingConfig: vi.fn(() => ({
|
||||
enabled: true,
|
||||
systemTtl: '1h',
|
||||
toolsTtl: '1h',
|
||||
historyTtl: '5m',
|
||||
cacheSystem: true,
|
||||
cacheTools: true,
|
||||
cacheHistory: true,
|
||||
vertexFallbackTo5m: false,
|
||||
})),
|
||||
};
|
||||
const runtimeWithDebug = new AiSdkKtxLlmRuntime({
|
||||
llmProvider: provider as any,
|
||||
debugRequestRecorder: { record },
|
||||
});
|
||||
|
||||
await runtimeWithDebug.runAgentLoop({
|
||||
modelRole: 'candidateExtraction',
|
||||
systemPrompt: 'SECRET SYSTEM PROMPT',
|
||||
userPrompt: 'SECRET USER PROMPT',
|
||||
toolSet: {
|
||||
emit_candidate: {
|
||||
description: 'SECRET TOOL DESCRIPTION',
|
||||
inputSchema: {},
|
||||
execute: vi.fn(),
|
||||
} as any,
|
||||
},
|
||||
stepBudget: 10,
|
||||
telemetryTags: { operationName: 'ingest-bundle-wu', source: 'metabase', jobId: 'job-1', unitKey: 'cards/1' },
|
||||
});
|
||||
|
||||
expect(record).toHaveBeenCalledTimes(1);
|
||||
expect(record).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
operationName: 'ingest-bundle-wu',
|
||||
source: 'metabase',
|
||||
jobId: 'job-1',
|
||||
unitKey: 'cards/1',
|
||||
modelRole: 'candidateExtraction',
|
||||
modelId: 'claude-sonnet-4-6',
|
||||
messageCount: 2,
|
||||
toolNames: ['emit_candidate'],
|
||||
}),
|
||||
);
|
||||
const providerOptions = record.mock.calls[0][0].providerOptions;
|
||||
expect(providerOptions).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ target: 'message', index: 0, role: 'system' }),
|
||||
expect.objectContaining({ target: 'message-part', index: 1, role: 'user', partIndex: 0 }),
|
||||
expect.objectContaining({ target: 'tool', name: 'emit_candidate' }),
|
||||
]),
|
||||
);
|
||||
expect(providerOptions).toHaveLength(3);
|
||||
const serialized = JSON.stringify(record.mock.calls[0][0]);
|
||||
expect(serialized).not.toContain('SECRET SYSTEM PROMPT');
|
||||
expect(serialized).not.toContain('SECRET USER PROMPT');
|
||||
expect(serialized).not.toContain('SECRET TOOL DESCRIPTION');
|
||||
});
|
||||
});
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import { CLAUDE_CODE_PROVIDER_ENV_DENYLIST, createKtxClaudeCodeEnv } from './claude-code-env.js';
|
||||
|
||||
describe('createKtxClaudeCodeEnv', () => {
|
||||
it('strips provider-routing credentials from the Claude Code child environment', () => {
|
||||
const seeded = Object.fromEntries(CLAUDE_CODE_PROVIDER_ENV_DENYLIST.map((key) => [key, `${key}-value`]));
|
||||
const env = createKtxClaudeCodeEnv({
|
||||
...seeded,
|
||||
PATH: '/usr/bin',
|
||||
HOME: '/Users/test',
|
||||
});
|
||||
|
||||
for (const key of CLAUDE_CODE_PROVIDER_ENV_DENYLIST) {
|
||||
expect(env).not.toHaveProperty(key);
|
||||
}
|
||||
expect(env.PATH).toBe('/usr/bin');
|
||||
expect(env.HOME).toBe('/Users/test');
|
||||
});
|
||||
});
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import { resolveClaudeCodeModel } from './claude-code-models.js';
|
||||
|
||||
describe('resolveClaudeCodeModel', () => {
|
||||
it.each([
|
||||
['sonnet', 'claude-sonnet-4-6'],
|
||||
['opus', 'claude-opus-4-7'],
|
||||
['haiku', 'claude-haiku-4-5'],
|
||||
['claude-sonnet-4-6', 'claude-sonnet-4-6'],
|
||||
])('maps %s to %s', (input, expected) => {
|
||||
expect(resolveClaudeCodeModel(input)).toBe(expected);
|
||||
});
|
||||
|
||||
it('rejects unsupported aliases', () => {
|
||||
expect(() => resolveClaudeCodeModel('gpt-5')).toThrow('Unsupported Claude Code model');
|
||||
});
|
||||
});
|
||||
|
|
@ -1,505 +0,0 @@
|
|||
import { describe, expect, it, vi } from 'vitest';
|
||||
import { z } from 'zod';
|
||||
import type { SDKMessage } from '@anthropic-ai/claude-agent-sdk';
|
||||
import { ClaudeCodeKtxLlmRuntime, mapClaudeCodeStopReason, runClaudeCodeAuthProbe } from './claude-code-runtime.js';
|
||||
|
||||
async function* stream(messages: SDKMessage[]): AsyncGenerator<SDKMessage, void> {
|
||||
for (const message of messages) {
|
||||
yield message;
|
||||
}
|
||||
}
|
||||
|
||||
function initMessage(overrides: Partial<Extract<SDKMessage, { type: 'system'; subtype: 'init' }>> = {}): Extract<
|
||||
SDKMessage,
|
||||
{ type: 'system'; subtype: 'init' }
|
||||
> {
|
||||
return {
|
||||
type: 'system',
|
||||
subtype: 'init',
|
||||
apiKeySource: 'none' as never, // pragma: allowlist secret
|
||||
claude_code_version: '0.3.142',
|
||||
cwd: '/tmp/project',
|
||||
tools: [],
|
||||
mcp_servers: [],
|
||||
model: 'claude-sonnet-4-6',
|
||||
permissionMode: 'dontAsk',
|
||||
slash_commands: [],
|
||||
output_style: 'default',
|
||||
skills: [],
|
||||
plugins: [],
|
||||
uuid: '00000000-0000-4000-8000-000000000001',
|
||||
session_id: 'session-id',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function resultMessage(overrides: Partial<Extract<SDKMessage, { type: 'result' }>> = {}): Extract<
|
||||
SDKMessage,
|
||||
{ type: 'result' }
|
||||
> {
|
||||
return {
|
||||
type: 'result',
|
||||
subtype: 'success',
|
||||
duration_ms: 1,
|
||||
duration_api_ms: 1,
|
||||
is_error: false,
|
||||
num_turns: 1,
|
||||
result: 'ok',
|
||||
stop_reason: null,
|
||||
total_cost_usd: 0,
|
||||
usage: {} as never,
|
||||
modelUsage: {},
|
||||
permission_denials: [],
|
||||
errors: [],
|
||||
uuid: '00000000-0000-4000-8000-000000000002',
|
||||
session_id: 'session-id',
|
||||
...overrides,
|
||||
} as Extract<SDKMessage, { type: 'result' }>;
|
||||
}
|
||||
|
||||
describe('ClaudeCodeKtxLlmRuntime', () => {
|
||||
it('passes isolation options and scrubbed env to text generation', async () => {
|
||||
const query = vi.fn((_input: any) => stream([initMessage(), resultMessage({ result: 'hello' })]));
|
||||
const runtime = new ClaudeCodeKtxLlmRuntime({
|
||||
projectDir: '/tmp/project',
|
||||
modelSlots: { default: 'sonnet' },
|
||||
query,
|
||||
env: { ANTHROPIC_API_KEY: 'sk-ant-test', PATH: '/usr/bin' }, // pragma: allowlist secret
|
||||
});
|
||||
|
||||
await expect(runtime.generateText({ role: 'default', prompt: 'say hello' })).resolves.toBe('hello');
|
||||
expect(query).toHaveBeenCalledWith({
|
||||
prompt: 'say hello',
|
||||
options: expect.objectContaining({
|
||||
cwd: '/tmp/project',
|
||||
model: 'claude-sonnet-4-6',
|
||||
maxTurns: 1,
|
||||
settingSources: [],
|
||||
skills: [],
|
||||
plugins: [],
|
||||
tools: [],
|
||||
managedSettings: {
|
||||
allowManagedMcpServersOnly: true,
|
||||
allowedMcpServers: [],
|
||||
},
|
||||
strictMcpConfig: true,
|
||||
allowedTools: [],
|
||||
permissionMode: 'dontAsk',
|
||||
persistSession: false,
|
||||
env: expect.not.objectContaining({ ANTHROPIC_API_KEY: 'sk-ant-test' }),
|
||||
}),
|
||||
});
|
||||
});
|
||||
|
||||
it('validates structured output with the caller schema and whitelists the SDK StructuredOutput tool', async () => {
|
||||
const schema = z.object({ answer: z.string() });
|
||||
const query = vi.fn((_input: any) =>
|
||||
stream([
|
||||
initMessage({ tools: ['StructuredOutput'] }),
|
||||
resultMessage({ structured_output: { answer: 'yes' } }),
|
||||
]),
|
||||
);
|
||||
const runtime = new ClaudeCodeKtxLlmRuntime({
|
||||
projectDir: '/tmp/project',
|
||||
modelSlots: { default: 'sonnet' },
|
||||
query,
|
||||
env: {},
|
||||
});
|
||||
|
||||
await expect(runtime.generateObject({ role: 'default', prompt: 'json', schema })).resolves.toEqual({ answer: 'yes' });
|
||||
expect(query.mock.calls[0][0].options.outputFormat).toMatchObject({
|
||||
type: 'json_schema',
|
||||
schema: expect.objectContaining({ type: 'object' }),
|
||||
});
|
||||
});
|
||||
|
||||
it('registers only exact KTX MCP tool ids and denies non-KTX tools', async () => {
|
||||
const query = vi.fn((_input: any) =>
|
||||
stream([
|
||||
initMessage({ tools: ['mcp__ktx__load_skill'], mcp_servers: [{ name: 'ktx', status: 'connected' }] }),
|
||||
{
|
||||
type: 'assistant',
|
||||
message: { role: 'assistant', content: [] },
|
||||
parent_tool_use_id: null,
|
||||
uuid: '00000000-0000-4000-8000-000000000003',
|
||||
session_id: 'session-id',
|
||||
} as unknown as SDKMessage,
|
||||
resultMessage({ subtype: 'error_max_turns', is_error: true }),
|
||||
]),
|
||||
);
|
||||
const runtime = new ClaudeCodeKtxLlmRuntime({
|
||||
projectDir: '/tmp/project',
|
||||
modelSlots: { default: 'sonnet' },
|
||||
query,
|
||||
env: {},
|
||||
});
|
||||
const onStepFinish = vi.fn();
|
||||
|
||||
await runtime.runAgentLoop({
|
||||
modelRole: 'default',
|
||||
systemPrompt: 'system',
|
||||
userPrompt: 'user',
|
||||
toolSet: {
|
||||
load_skill: {
|
||||
name: 'load_skill',
|
||||
description: 'Load skill.',
|
||||
inputSchema: z.object({ name: z.string() }),
|
||||
execute: async () => ({ markdown: 'loaded' }),
|
||||
},
|
||||
},
|
||||
stepBudget: 1,
|
||||
telemetryTags: { operationName: 'test' },
|
||||
onStepFinish,
|
||||
});
|
||||
|
||||
const options = query.mock.calls[0][0].options;
|
||||
expect(options.allowedTools).toEqual(['mcp__ktx__load_skill']);
|
||||
expect(options.managedSettings).toEqual({
|
||||
allowManagedMcpServersOnly: true,
|
||||
allowedMcpServers: [{ serverName: 'ktx' }],
|
||||
});
|
||||
expect(options.strictMcpConfig).toBe(true);
|
||||
expect(await options.canUseTool('mcp__ktx__load_skill', {}, { signal: new AbortController().signal, toolUseID: '1' })).toEqual({
|
||||
behavior: 'allow',
|
||||
toolUseID: '1',
|
||||
});
|
||||
expect(await options.canUseTool('Bash', {}, { signal: new AbortController().signal, toolUseID: '2' })).toMatchObject({
|
||||
behavior: 'deny',
|
||||
toolUseID: '2',
|
||||
});
|
||||
expect(onStepFinish).toHaveBeenCalledWith({ stepIndex: 1, stepBudget: 1 });
|
||||
});
|
||||
|
||||
it('treats host-discovered commands skills and agents as non-fatal init metadata for text and auth probe', async () => {
|
||||
const hostDiscoveredInit = initMessage({
|
||||
slash_commands: ['/help', '/compact', '/clear', '/user-command'],
|
||||
skills: ['pdf', 'docx'],
|
||||
agents: ['claude', 'Explore', 'general-purpose'],
|
||||
});
|
||||
const textQuery = vi.fn((_input: any) => stream([hostDiscoveredInit, resultMessage({ result: 'hello' })]));
|
||||
const runtime = new ClaudeCodeKtxLlmRuntime({
|
||||
projectDir: '/tmp/project',
|
||||
modelSlots: { default: 'sonnet' },
|
||||
query: textQuery,
|
||||
env: { ANTHROPIC_API_KEY: 'sk-ant-test', PATH: '/usr/bin' }, // pragma: allowlist secret
|
||||
});
|
||||
|
||||
await expect(runtime.generateText({ role: 'default', prompt: 'say hello' })).resolves.toBe('hello');
|
||||
const textOptions = textQuery.mock.calls[0][0].options;
|
||||
expect(textOptions).toMatchObject({
|
||||
settingSources: [],
|
||||
skills: [],
|
||||
plugins: [],
|
||||
tools: [],
|
||||
managedSettings: {
|
||||
allowManagedMcpServersOnly: true,
|
||||
allowedMcpServers: [],
|
||||
},
|
||||
strictMcpConfig: true,
|
||||
allowedTools: [],
|
||||
permissionMode: 'dontAsk',
|
||||
persistSession: false,
|
||||
env: expect.not.objectContaining({ ANTHROPIC_API_KEY: 'sk-ant-test' }),
|
||||
});
|
||||
expect(textOptions.disallowedTools).toEqual(expect.arrayContaining(['Agent', 'Task', 'Bash']));
|
||||
expect(await textOptions.canUseTool('Agent', {}, { signal: new AbortController().signal, toolUseID: 'agent' })).toMatchObject({
|
||||
behavior: 'deny',
|
||||
toolUseID: 'agent',
|
||||
});
|
||||
expect(await textOptions.canUseTool('Skill', {}, { signal: new AbortController().signal, toolUseID: 'skill' })).toMatchObject({
|
||||
behavior: 'deny',
|
||||
toolUseID: 'skill',
|
||||
});
|
||||
expect(
|
||||
await textOptions.canUseTool('SlashCommand', {}, { signal: new AbortController().signal, toolUseID: 'slash' }),
|
||||
).toMatchObject({
|
||||
behavior: 'deny',
|
||||
toolUseID: 'slash',
|
||||
});
|
||||
|
||||
const probeQuery = vi.fn((_input: any) => stream([hostDiscoveredInit, resultMessage({ result: 'ok' })]));
|
||||
await expect(
|
||||
runClaudeCodeAuthProbe({
|
||||
projectDir: '/tmp/project',
|
||||
model: 'sonnet',
|
||||
query: probeQuery,
|
||||
env: { ANTHROPIC_AUTH_TOKEN: 'token', HOME: '/Users/test' },
|
||||
}),
|
||||
).resolves.toEqual({ ok: true });
|
||||
expect(probeQuery.mock.calls[0][0].options).toMatchObject({
|
||||
settingSources: [],
|
||||
skills: [],
|
||||
plugins: [],
|
||||
tools: [],
|
||||
allowedTools: [],
|
||||
permissionMode: 'dontAsk',
|
||||
persistSession: false,
|
||||
env: expect.objectContaining({ HOME: '/Users/test' }),
|
||||
});
|
||||
expect(probeQuery.mock.calls[0][0].options.env).not.toEqual(
|
||||
expect.objectContaining({ ANTHROPIC_AUTH_TOKEN: 'token' }),
|
||||
);
|
||||
});
|
||||
|
||||
it('allows host-discovered context during agent loops while requiring exact KTX MCP tools and servers', async () => {
|
||||
const query = vi.fn((_input: any) =>
|
||||
stream([
|
||||
initMessage({
|
||||
tools: ['mcp__ktx__load_skill'],
|
||||
mcp_servers: [{ name: 'ktx', status: 'connected' }],
|
||||
slash_commands: ['/help', '/compact', '/clear'],
|
||||
skills: ['memory-agent', 'doc-reader'],
|
||||
agents: ['claude', 'Plan', 'Explore'],
|
||||
}),
|
||||
{
|
||||
type: 'assistant',
|
||||
message: { role: 'assistant', content: [] },
|
||||
parent_tool_use_id: null,
|
||||
uuid: '00000000-0000-4000-8000-000000000006',
|
||||
session_id: 'session-id',
|
||||
} as unknown as SDKMessage,
|
||||
resultMessage({ subtype: 'error_max_turns', is_error: true }),
|
||||
]),
|
||||
);
|
||||
const runtime = new ClaudeCodeKtxLlmRuntime({
|
||||
projectDir: '/tmp/project',
|
||||
modelSlots: { default: 'sonnet' },
|
||||
query,
|
||||
env: {},
|
||||
});
|
||||
|
||||
await expect(
|
||||
runtime.runAgentLoop({
|
||||
modelRole: 'default',
|
||||
systemPrompt: 'system',
|
||||
userPrompt: 'user',
|
||||
toolSet: {
|
||||
load_skill: {
|
||||
name: 'load_skill',
|
||||
description: 'Load skill.',
|
||||
inputSchema: z.object({ name: z.string() }),
|
||||
execute: async () => ({ markdown: 'loaded' }),
|
||||
},
|
||||
},
|
||||
stepBudget: 1,
|
||||
telemetryTags: { operationName: 'test' },
|
||||
}),
|
||||
).resolves.toEqual({ stopReason: 'budget' });
|
||||
|
||||
const options = query.mock.calls[0][0].options;
|
||||
expect(options.allowedTools).toEqual(['mcp__ktx__load_skill']);
|
||||
expect(options.managedSettings).toEqual({
|
||||
allowManagedMcpServersOnly: true,
|
||||
allowedMcpServers: [{ serverName: 'ktx' }],
|
||||
});
|
||||
expect(options.strictMcpConfig).toBe(true);
|
||||
expect(await options.canUseTool('mcp__ktx__load_skill', {}, { signal: new AbortController().signal, toolUseID: '1' })).toEqual({
|
||||
behavior: 'allow',
|
||||
toolUseID: '1',
|
||||
});
|
||||
expect(await options.canUseTool('Task', {}, { signal: new AbortController().signal, toolUseID: '2' })).toMatchObject({
|
||||
behavior: 'deny',
|
||||
toolUseID: '2',
|
||||
});
|
||||
expect(await options.canUseTool('Skill', {}, { signal: new AbortController().signal, toolUseID: '3' })).toMatchObject({
|
||||
behavior: 'deny',
|
||||
toolUseID: '3',
|
||||
});
|
||||
});
|
||||
|
||||
it('still rejects unexpected tools, missing KTX tools, plugins, and non-KTX MCP servers from init messages', async () => {
|
||||
const query = vi.fn((_input: any) =>
|
||||
stream([
|
||||
initMessage({
|
||||
tools: ['Bash'],
|
||||
mcp_servers: [{ name: 'filesystem', status: 'connected' }],
|
||||
plugins: [{ name: 'host-plugin', path: '/tmp/plugin' }],
|
||||
}),
|
||||
resultMessage({ result: 'hello' }),
|
||||
]),
|
||||
);
|
||||
const runtime = new ClaudeCodeKtxLlmRuntime({
|
||||
projectDir: '/tmp/project',
|
||||
modelSlots: { default: 'sonnet' },
|
||||
query,
|
||||
env: {},
|
||||
});
|
||||
|
||||
await expect(
|
||||
runtime.generateText({
|
||||
role: 'default',
|
||||
prompt: 'say hello',
|
||||
tools: {
|
||||
load_skill: {
|
||||
name: 'load_skill',
|
||||
description: 'Load skill.',
|
||||
inputSchema: z.object({ name: z.string() }),
|
||||
execute: async () => ({ markdown: 'loaded' }),
|
||||
},
|
||||
},
|
||||
}),
|
||||
).rejects.toThrow(
|
||||
/Claude Code runtime isolation failed: .*tools=Bash.*missing_tools=mcp__ktx__load_skill.*mcp_servers=filesystem.*plugins=host-plugin/,
|
||||
);
|
||||
});
|
||||
|
||||
it('passes scrubbed env to object generation and agent loops', async () => {
|
||||
const schema = z.object({ answer: z.string() });
|
||||
const objectQuery = vi.fn((_input: any) =>
|
||||
stream([
|
||||
initMessage({ tools: ['StructuredOutput'] }),
|
||||
resultMessage({ structured_output: { answer: 'yes' } }),
|
||||
]),
|
||||
);
|
||||
const objectRuntime = new ClaudeCodeKtxLlmRuntime({
|
||||
projectDir: '/tmp/project',
|
||||
modelSlots: { default: 'sonnet' },
|
||||
query: objectQuery,
|
||||
env: { ANTHROPIC_API_KEY: 'sk-ant-test', AWS_PROFILE: 'prod', PATH: '/usr/bin' }, // pragma: allowlist secret
|
||||
});
|
||||
|
||||
await expect(objectRuntime.generateObject({ role: 'default', prompt: 'json', schema })).resolves.toEqual({
|
||||
answer: 'yes',
|
||||
});
|
||||
expect(objectQuery.mock.calls[0][0].options.env).toEqual(expect.objectContaining({ PATH: '/usr/bin' }));
|
||||
expect(objectQuery.mock.calls[0][0].options.managedSettings).toEqual({
|
||||
allowManagedMcpServersOnly: true,
|
||||
allowedMcpServers: [],
|
||||
});
|
||||
expect(objectQuery.mock.calls[0][0].options.env).not.toEqual(
|
||||
expect.objectContaining({ ANTHROPIC_API_KEY: 'sk-ant-test', AWS_PROFILE: 'prod' }), // pragma: allowlist secret
|
||||
);
|
||||
|
||||
const agentQuery = vi.fn((_input: any) =>
|
||||
stream([
|
||||
initMessage({ tools: ['mcp__ktx__load_skill'], mcp_servers: [{ name: 'ktx', status: 'connected' }] }),
|
||||
{
|
||||
type: 'assistant',
|
||||
message: { role: 'assistant', content: [] },
|
||||
parent_tool_use_id: null,
|
||||
uuid: '00000000-0000-4000-8000-000000000004',
|
||||
session_id: 'session-id',
|
||||
} as unknown as SDKMessage,
|
||||
resultMessage({ subtype: 'error_max_turns', is_error: true }),
|
||||
]),
|
||||
);
|
||||
const agentRuntime = new ClaudeCodeKtxLlmRuntime({
|
||||
projectDir: '/tmp/project',
|
||||
modelSlots: { default: 'sonnet' },
|
||||
query: agentQuery,
|
||||
env: { ANTHROPIC_AUTH_TOKEN: 'token', CLAUDE_CODE_USE_VERTEX: '1', HOME: '/Users/test' },
|
||||
});
|
||||
|
||||
await agentRuntime.runAgentLoop({
|
||||
modelRole: 'default',
|
||||
systemPrompt: 'system',
|
||||
userPrompt: 'user',
|
||||
toolSet: {
|
||||
load_skill: {
|
||||
name: 'load_skill',
|
||||
description: 'Load skill.',
|
||||
inputSchema: z.object({ name: z.string() }),
|
||||
execute: async () => ({ markdown: 'loaded' }),
|
||||
},
|
||||
},
|
||||
stepBudget: 1,
|
||||
telemetryTags: { operationName: 'test' },
|
||||
});
|
||||
expect(agentQuery.mock.calls[0][0].options.env).toEqual(expect.objectContaining({ HOME: '/Users/test' }));
|
||||
expect(agentQuery.mock.calls[0][0].options.managedSettings).toEqual({
|
||||
allowManagedMcpServersOnly: true,
|
||||
allowedMcpServers: [{ serverName: 'ktx' }],
|
||||
});
|
||||
expect(agentQuery.mock.calls[0][0].options.env).not.toEqual(
|
||||
expect.objectContaining({ ANTHROPIC_AUTH_TOKEN: 'token', CLAUDE_CODE_USE_VERTEX: '1' }),
|
||||
);
|
||||
});
|
||||
|
||||
it('logs and ignores onStepFinish callback errors', async () => {
|
||||
const query = vi.fn((_input: any) =>
|
||||
stream([
|
||||
initMessage(),
|
||||
{
|
||||
type: 'assistant',
|
||||
message: { role: 'assistant', content: [] },
|
||||
parent_tool_use_id: null,
|
||||
uuid: '00000000-0000-4000-8000-000000000005',
|
||||
session_id: 'session-id',
|
||||
} as unknown as SDKMessage,
|
||||
resultMessage({ subtype: 'success', terminal_reason: 'completed' }),
|
||||
]),
|
||||
);
|
||||
const logger = {
|
||||
debug: vi.fn(),
|
||||
log: vi.fn(),
|
||||
warn: vi.fn(),
|
||||
error: vi.fn(),
|
||||
};
|
||||
const runtime = new ClaudeCodeKtxLlmRuntime({
|
||||
projectDir: '/tmp/project',
|
||||
modelSlots: { default: 'sonnet' },
|
||||
query,
|
||||
env: {},
|
||||
logger,
|
||||
});
|
||||
|
||||
await expect(
|
||||
runtime.runAgentLoop({
|
||||
modelRole: 'default',
|
||||
systemPrompt: 'system',
|
||||
userPrompt: 'user',
|
||||
toolSet: {},
|
||||
stepBudget: 1,
|
||||
telemetryTags: { operationName: 'test' },
|
||||
onStepFinish: async () => {
|
||||
throw new Error('callback exploded');
|
||||
},
|
||||
}),
|
||||
).resolves.toEqual({ stopReason: 'natural' });
|
||||
expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('callback exploded'));
|
||||
});
|
||||
|
||||
it('maps max-turn terminal reasons to budget', () => {
|
||||
expect(mapClaudeCodeStopReason(resultMessage({ subtype: 'error_max_turns' }))).toBe('budget');
|
||||
expect(mapClaudeCodeStopReason(resultMessage({ terminal_reason: 'max_turns' }))).toBe('budget');
|
||||
expect(mapClaudeCodeStopReason(resultMessage({ stop_reason: 'max_turns' }))).toBe('budget');
|
||||
expect(mapClaudeCodeStopReason(resultMessage({ subtype: 'success', terminal_reason: 'completed' }))).toBe('natural');
|
||||
expect(mapClaudeCodeStopReason(resultMessage({ subtype: 'error_during_execution' }))).toBe('error');
|
||||
});
|
||||
|
||||
it('auth probe uses isolation options and a scrubbed env', async () => {
|
||||
const query = vi.fn((_input: any) => stream([initMessage(), resultMessage({ result: 'ok' })]));
|
||||
|
||||
await expect(
|
||||
runClaudeCodeAuthProbe({ projectDir: '/tmp/project', model: 'sonnet', query, env: { ANTHROPIC_API_KEY: 'sk-ant-test' } }), // pragma: allowlist secret
|
||||
).resolves.toEqual({ ok: true });
|
||||
expect(query.mock.calls[0][0].options).toMatchObject({
|
||||
settingSources: [],
|
||||
skills: [],
|
||||
plugins: [],
|
||||
tools: [],
|
||||
managedSettings: {
|
||||
allowManagedMcpServersOnly: true,
|
||||
allowedMcpServers: [],
|
||||
},
|
||||
strictMcpConfig: true,
|
||||
allowedTools: [],
|
||||
persistSession: false,
|
||||
env: expect.not.objectContaining({ ANTHROPIC_API_KEY: 'sk-ant-test' }),
|
||||
});
|
||||
});
|
||||
|
||||
it('reports unsupported Claude Code models without framing them as auth failures', async () => {
|
||||
await expect(
|
||||
runClaudeCodeAuthProbe({
|
||||
projectDir: '/tmp/project',
|
||||
model: 'gpt-5',
|
||||
query: vi.fn(),
|
||||
env: {},
|
||||
}),
|
||||
).resolves.toEqual({
|
||||
ok: false,
|
||||
message: 'Unsupported Claude Code model "gpt-5". Use sonnet, opus, haiku, or a claude-* model id.',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -1,123 +0,0 @@
|
|||
import { mkdtemp, readFile, rm } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, describe, expect, it } from 'vitest';
|
||||
import {
|
||||
createJsonlKtxLlmDebugRequestRecorder,
|
||||
summarizeKtxLlmDebugRequest,
|
||||
} from './debug-request-recorder.js';
|
||||
|
||||
describe('summarizeKtxLlmDebugRequest', () => {
|
||||
it('records providerOptions positions without message text or tool schemas', () => {
|
||||
const summary = summarizeKtxLlmDebugRequest({
|
||||
operationName: 'ingest-bundle-wu',
|
||||
source: 'metabase',
|
||||
jobId: 'job-1',
|
||||
unitKey: 'cards/1',
|
||||
modelRole: 'candidateExtraction',
|
||||
modelId: 'claude-sonnet-4-6',
|
||||
messages: [
|
||||
{
|
||||
role: 'system',
|
||||
content: 'SECRET SYSTEM PROMPT',
|
||||
providerOptions: { anthropic: { cacheControl: { type: 'ephemeral', ttl: '1h' } } },
|
||||
},
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: 'SECRET USER PROMPT',
|
||||
providerOptions: { anthropic: { cacheControl: { type: 'ephemeral', ttl: '5m' } } },
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
tools: {
|
||||
emit_candidate: {
|
||||
description: 'SECRET TOOL DESCRIPTION',
|
||||
inputSchema: { secret: true },
|
||||
providerOptions: { anthropic: { cacheControl: { type: 'ephemeral', ttl: '1h' } } },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(summary).toMatchObject({
|
||||
operationName: 'ingest-bundle-wu',
|
||||
source: 'metabase',
|
||||
jobId: 'job-1',
|
||||
unitKey: 'cards/1',
|
||||
modelRole: 'candidateExtraction',
|
||||
modelId: 'claude-sonnet-4-6',
|
||||
messageCount: 2,
|
||||
toolNames: ['emit_candidate'],
|
||||
providerOptions: [
|
||||
{
|
||||
target: 'message',
|
||||
index: 0,
|
||||
role: 'system',
|
||||
providerOptions: { anthropic: { cacheControl: { type: 'ephemeral', ttl: '1h' } } },
|
||||
},
|
||||
{
|
||||
target: 'message-part',
|
||||
index: 1,
|
||||
role: 'user',
|
||||
partIndex: 0,
|
||||
providerOptions: { anthropic: { cacheControl: { type: 'ephemeral', ttl: '5m' } } },
|
||||
},
|
||||
{
|
||||
target: 'tool',
|
||||
name: 'emit_candidate',
|
||||
providerOptions: { anthropic: { cacheControl: { type: 'ephemeral', ttl: '1h' } } },
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const serialized = JSON.stringify(summary);
|
||||
expect(serialized).not.toContain('SECRET SYSTEM PROMPT');
|
||||
expect(serialized).not.toContain('SECRET USER PROMPT');
|
||||
expect(serialized).not.toContain('SECRET TOOL DESCRIPTION');
|
||||
expect(serialized).not.toContain('inputSchema');
|
||||
});
|
||||
});
|
||||
|
||||
describe('createJsonlKtxLlmDebugRequestRecorder', () => {
|
||||
let tempDir: string | undefined;
|
||||
|
||||
afterEach(async () => {
|
||||
if (tempDir) {
|
||||
await rm(tempDir, { recursive: true, force: true });
|
||||
tempDir = undefined;
|
||||
}
|
||||
});
|
||||
|
||||
it('appends one JSON object per recorded request', async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'ktx-llm-debug-'));
|
||||
const filePath = join(tempDir, 'nested', 'llm-debug.jsonl');
|
||||
const recorder = createJsonlKtxLlmDebugRequestRecorder(filePath);
|
||||
|
||||
await recorder.record({
|
||||
timestamp: '2026-05-04T00:00:00.000Z',
|
||||
operationName: 'ingest-bundle-wu',
|
||||
modelRole: 'candidateExtraction',
|
||||
modelId: 'claude-sonnet-4-6',
|
||||
messageCount: 2,
|
||||
toolNames: ['emit_candidate'],
|
||||
providerOptions: [],
|
||||
});
|
||||
await recorder.record({
|
||||
timestamp: '2026-05-04T00:00:01.000Z',
|
||||
operationName: 'ingest-bundle-reconcile',
|
||||
modelRole: 'reconcile',
|
||||
modelId: 'claude-sonnet-4-6',
|
||||
messageCount: 2,
|
||||
toolNames: [],
|
||||
providerOptions: [],
|
||||
});
|
||||
|
||||
const lines = (await readFile(filePath, 'utf8')).trim().split('\n').map((line) => JSON.parse(line));
|
||||
expect(lines).toHaveLength(2);
|
||||
expect(lines[0]).toMatchObject({ operationName: 'ingest-bundle-wu', modelRole: 'candidateExtraction' });
|
||||
expect(lines[1]).toMatchObject({ operationName: 'ingest-bundle-reconcile', modelRole: 'reconcile' });
|
||||
});
|
||||
});
|
||||
|
|
@ -1,38 +0,0 @@
|
|||
import { describe, expect, it, vi } from 'vitest';
|
||||
import { KtxIngestEmbeddingPortAdapter, KtxScanEmbeddingPortAdapter } from './embedding-port.js';
|
||||
|
||||
describe('KTX embedding port adapters', () => {
|
||||
it('adapts LLM modules embeddings to ingest embedding port shape', async () => {
|
||||
const provider = {
|
||||
dimensions: 3,
|
||||
maxBatchSize: 2,
|
||||
embed: vi.fn(async () => [1, 2, 3]),
|
||||
[['embed', 'Many'].join('')]: vi.fn(async () => [
|
||||
[1, 2, 3],
|
||||
[4, 5, 6],
|
||||
]),
|
||||
};
|
||||
const adapter = new KtxIngestEmbeddingPortAdapter(provider as never);
|
||||
|
||||
await expect(adapter.computeEmbedding('alpha')).resolves.toEqual([1, 2, 3]);
|
||||
await expect(adapter.computeEmbeddingsBulk(['alpha', 'beta'])).resolves.toEqual([
|
||||
[1, 2, 3],
|
||||
[4, 5, 6],
|
||||
]);
|
||||
expect(adapter.maxBatchSize).toBe(2);
|
||||
});
|
||||
|
||||
it('adapts LLM modules embeddings to scan embedding port shape', async () => {
|
||||
const provider = {
|
||||
dimensions: 3,
|
||||
maxBatchSize: 2,
|
||||
embed: vi.fn(),
|
||||
[['embed', 'Many'].join('')]: vi.fn(async () => [[1, 2, 3]]),
|
||||
};
|
||||
const adapter = new KtxScanEmbeddingPortAdapter(provider as never);
|
||||
|
||||
await expect(adapter.embedBatch(['alpha'])).resolves.toEqual([[1, 2, 3]]);
|
||||
expect(adapter.dimensions).toBe(3);
|
||||
expect(adapter.maxBatchSize).toBe(2);
|
||||
});
|
||||
});
|
||||
|
|
@ -1,211 +0,0 @@
|
|||
import { describe, expect, it, vi } from 'vitest';
|
||||
import {
|
||||
buildDefaultKtxProjectConfig,
|
||||
type KtxProjectEmbeddingConfig,
|
||||
type KtxProjectLlmConfig,
|
||||
} from '../project/config.js';
|
||||
import {
|
||||
createLocalKtxEmbeddingProviderFromConfig,
|
||||
createLocalKtxLlmProviderFromConfig,
|
||||
resolveLocalKtxEmbeddingConfig,
|
||||
resolveLocalKtxLlmConfig,
|
||||
} from './local-config.js';
|
||||
|
||||
describe('local KTX LLM config', () => {
|
||||
it('resolves env and file references into a KtxLlmConfig', () => {
|
||||
const config: KtxProjectLlmConfig = {
|
||||
provider: {
|
||||
backend: 'gateway',
|
||||
gateway: { api_key: 'env:AI_GATEWAY_API_KEY', base_url: 'https://gateway.example/v1' }, // pragma: allowlist secret
|
||||
},
|
||||
models: { default: 'env:KTX_MODEL', triage: 'anthropic/claude-haiku-4-5' },
|
||||
promptCaching: { enabled: false },
|
||||
};
|
||||
|
||||
expect(
|
||||
resolveLocalKtxLlmConfig(config, {
|
||||
AI_GATEWAY_API_KEY: 'gateway-key', // pragma: allowlist secret
|
||||
KTX_MODEL: 'anthropic/claude-sonnet-4-6',
|
||||
}),
|
||||
).toEqual({
|
||||
backend: 'gateway',
|
||||
gateway: { apiKey: 'gateway-key', baseURL: 'https://gateway.example/v1' }, // pragma: allowlist secret
|
||||
modelSlots: { default: 'anthropic/claude-sonnet-4-6', triage: 'anthropic/claude-haiku-4-5' },
|
||||
promptCaching: { enabled: false },
|
||||
});
|
||||
});
|
||||
|
||||
it('resolves Vertex AI env references into a KtxLlmConfig', () => {
|
||||
const config: KtxProjectLlmConfig = {
|
||||
provider: {
|
||||
backend: 'vertex',
|
||||
vertex: { project: 'env:GOOGLE_VERTEX_PROJECT', location: 'env:GOOGLE_VERTEX_LOCATION' },
|
||||
},
|
||||
models: { default: 'env:KTX_MODEL' },
|
||||
promptCaching: { enabled: true, vertexFallbackTo5m: true },
|
||||
};
|
||||
|
||||
expect(
|
||||
resolveLocalKtxLlmConfig(config, {
|
||||
GOOGLE_VERTEX_PROJECT: 'local-gcp-project',
|
||||
GOOGLE_VERTEX_LOCATION: 'us-east5',
|
||||
KTX_MODEL: 'claude-sonnet-4-6',
|
||||
}),
|
||||
).toEqual({
|
||||
backend: 'vertex',
|
||||
vertex: { project: 'local-gcp-project', location: 'us-east5' },
|
||||
modelSlots: { default: 'claude-sonnet-4-6' },
|
||||
promptCaching: { enabled: true, vertexFallbackTo5m: true },
|
||||
});
|
||||
});
|
||||
|
||||
it('ignores inactive Vertex AI references for non-Vertex backends', () => {
|
||||
const config: KtxProjectLlmConfig = {
|
||||
provider: {
|
||||
backend: 'anthropic',
|
||||
anthropic: { api_key: 'env:ANTHROPIC_API_KEY' }, // pragma: allowlist secret
|
||||
vertex: { location: 'env:MISSING_VERTEX_LOCATION' },
|
||||
},
|
||||
models: { default: 'claude-sonnet-4-6' },
|
||||
};
|
||||
|
||||
expect(
|
||||
resolveLocalKtxLlmConfig(config, {
|
||||
ANTHROPIC_API_KEY: 'sk-ant-test', // pragma: allowlist secret
|
||||
}),
|
||||
).toEqual({
|
||||
backend: 'anthropic',
|
||||
anthropic: { apiKey: 'sk-ant-test' }, // pragma: allowlist secret
|
||||
modelSlots: { default: 'claude-sonnet-4-6' },
|
||||
promptCaching: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
it('returns null when the local LLM backend is disabled', () => {
|
||||
expect(
|
||||
createLocalKtxLlmProviderFromConfig({
|
||||
provider: { backend: 'none' },
|
||||
models: {},
|
||||
}),
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it('constructs providers through LLM modules', () => {
|
||||
const createKtxLlmProvider = vi.fn(() => ({ getModel: vi.fn() }) as never);
|
||||
const result = createLocalKtxLlmProviderFromConfig(
|
||||
{
|
||||
provider: {
|
||||
backend: 'anthropic',
|
||||
anthropic: { api_key: 'env:ANTHROPIC_API_KEY' }, // pragma: allowlist secret
|
||||
},
|
||||
models: { default: 'claude-sonnet-4-6' },
|
||||
},
|
||||
{ env: { ANTHROPIC_API_KEY: 'sk-ant-test' }, createKtxLlmProvider }, // pragma: allowlist secret
|
||||
);
|
||||
|
||||
expect(result).not.toBeNull();
|
||||
expect(createKtxLlmProvider).toHaveBeenCalledWith({
|
||||
backend: 'anthropic',
|
||||
anthropic: { apiKey: 'sk-ant-test' }, // pragma: allowlist secret
|
||||
modelSlots: { default: 'claude-sonnet-4-6' },
|
||||
promptCaching: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
it('inherits enabled prompt caching from LLM modules when local config omits promptCaching', () => {
|
||||
const provider = createLocalKtxLlmProviderFromConfig({
|
||||
provider: {
|
||||
backend: 'gateway',
|
||||
gateway: { base_url: 'https://gateway.example/v1' },
|
||||
},
|
||||
models: { default: 'anthropic/claude-sonnet-4-6' },
|
||||
});
|
||||
|
||||
expect(provider?.promptCachingConfig()).toMatchObject({
|
||||
enabled: true,
|
||||
systemTtl: '1h',
|
||||
toolsTtl: '1h',
|
||||
historyTtl: '5m',
|
||||
vertexFallbackTo5m: false,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('local KTX embedding config', () => {
|
||||
it('resolves sentence-transformers config', () => {
|
||||
const config: KtxProjectEmbeddingConfig = {
|
||||
backend: 'sentence-transformers',
|
||||
model: 'all-MiniLM-L6-v2',
|
||||
dimensions: 384,
|
||||
sentenceTransformers: { base_url: 'http://localhost:18081', pathPrefix: '' },
|
||||
batchSize: 16,
|
||||
};
|
||||
|
||||
expect(resolveLocalKtxEmbeddingConfig(config, {})).toEqual({
|
||||
backend: 'sentence-transformers',
|
||||
model: 'all-MiniLM-L6-v2',
|
||||
dimensions: 384,
|
||||
sentenceTransformers: { baseURL: 'http://localhost:18081', pathPrefix: '' },
|
||||
batchSize: 16,
|
||||
});
|
||||
});
|
||||
|
||||
it('returns null when sentence-transformers has no base_url (managed daemon delegation)', () => {
|
||||
const config: KtxProjectEmbeddingConfig = {
|
||||
backend: 'sentence-transformers',
|
||||
model: 'all-MiniLM-L6-v2',
|
||||
dimensions: 384,
|
||||
sentenceTransformers: {
|
||||
base_url: '',
|
||||
pathPrefix: '',
|
||||
},
|
||||
};
|
||||
|
||||
expect(resolveLocalKtxEmbeddingConfig(config, {})).toBeNull();
|
||||
});
|
||||
|
||||
it('returns null when backend is openai but no apiKey is resolvable from env', () => {
|
||||
const config: KtxProjectEmbeddingConfig = {
|
||||
backend: 'openai',
|
||||
model: 'text-embedding-3-small',
|
||||
dimensions: 1536,
|
||||
openai: { api_key: 'env:OPENAI_API_KEY' }, // pragma: allowlist secret
|
||||
};
|
||||
|
||||
expect(resolveLocalKtxEmbeddingConfig(config, {})).toBeNull();
|
||||
});
|
||||
|
||||
it('resolves openai embedding config from env', () => {
|
||||
const config: KtxProjectEmbeddingConfig = {
|
||||
backend: 'openai',
|
||||
model: 'text-embedding-3-small',
|
||||
dimensions: 1536,
|
||||
openai: { api_key: 'env:OPENAI_API_KEY' }, // pragma: allowlist secret
|
||||
};
|
||||
|
||||
expect(
|
||||
resolveLocalKtxEmbeddingConfig(config, { OPENAI_API_KEY: 'sk-test' }), // pragma: allowlist secret
|
||||
).toEqual({
|
||||
backend: 'openai',
|
||||
model: 'text-embedding-3-small',
|
||||
dimensions: 1536,
|
||||
openai: { apiKey: 'sk-test' }, // pragma: allowlist secret
|
||||
batchSize: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
it('returns null for the default disabled project embedding config', () => {
|
||||
const createKtxEmbeddingProvider = vi.fn(() => ({}) as never);
|
||||
const provider = createLocalKtxEmbeddingProviderFromConfig(
|
||||
buildDefaultKtxProjectConfig().ingest.embeddings,
|
||||
{ createKtxEmbeddingProvider },
|
||||
);
|
||||
|
||||
expect(provider).toBeNull();
|
||||
expect(createKtxEmbeddingProvider).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('returns null when embeddings are disabled', () => {
|
||||
expect(createLocalKtxEmbeddingProviderFromConfig({ backend: 'none', dimensions: 8 })).toBeNull();
|
||||
});
|
||||
});
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
import { describe, expect, it, vi } from 'vitest';
|
||||
import { createLocalKtxLlmProviderFromConfig, createLocalKtxLlmRuntimeFromConfig } from './local-config.js';
|
||||
|
||||
describe('local KTX LLM runtime config', () => {
|
||||
it('creates a Claude Code runtime for claude-code backend without creating an AI SDK provider', () => {
|
||||
const runtime = createLocalKtxLlmRuntimeFromConfig(
|
||||
{
|
||||
provider: { backend: 'claude-code' },
|
||||
models: { default: 'sonnet', triage: 'haiku' },
|
||||
},
|
||||
{ env: {}, projectDir: '/tmp/project', createClaudeCodeRuntime: vi.fn((deps) => ({ deps }) as never) },
|
||||
);
|
||||
|
||||
expect(runtime).toMatchObject({ deps: expect.objectContaining({ projectDir: '/tmp/project' }) });
|
||||
});
|
||||
|
||||
it('returns null from the AI SDK provider factory for claude-code backend', () => {
|
||||
expect(
|
||||
createLocalKtxLlmProviderFromConfig({
|
||||
provider: { backend: 'claude-code' },
|
||||
models: { default: 'sonnet' },
|
||||
}),
|
||||
).toBeNull();
|
||||
});
|
||||
});
|
||||
|
|
@ -1,43 +0,0 @@
|
|||
import { describe, expect, it, vi } from 'vitest';
|
||||
import { z } from 'zod';
|
||||
import { createAiSdkToolSet, createClaudeSdkTools, normalizeKtxRuntimeToolOutput } from './runtime-tools.js';
|
||||
import type { KtxRuntimeToolDescriptor } from './runtime-port.js';
|
||||
|
||||
describe('runtime tool descriptors', () => {
|
||||
const descriptor: KtxRuntimeToolDescriptor<{ id: string }, { ok: boolean }> = {
|
||||
name: 'read_thing',
|
||||
description: 'Read one thing.',
|
||||
inputSchema: z.object({ id: z.string() }),
|
||||
execute: vi.fn(async (input) => ({
|
||||
markdown: `Read ${input.id}`,
|
||||
structured: { ok: true },
|
||||
})),
|
||||
};
|
||||
|
||||
it('normalizes string and object tool outputs into markdown plus optional structured payload', () => {
|
||||
expect(normalizeKtxRuntimeToolOutput('plain text')).toEqual({ markdown: 'plain text' });
|
||||
expect(normalizeKtxRuntimeToolOutput({ markdown: 'shown', structured: { id: 1 } })).toEqual({
|
||||
markdown: 'shown',
|
||||
structured: { id: 1 },
|
||||
});
|
||||
expect(normalizeKtxRuntimeToolOutput({ name: 'skill', content: 'body' })).toEqual({
|
||||
markdown: '```json\n{\n "name": "skill",\n "content": "body"\n}\n```',
|
||||
structured: { name: 'skill', content: 'body' },
|
||||
});
|
||||
});
|
||||
|
||||
it('builds AI SDK tools that expose markdown to the model', async () => {
|
||||
const tools = createAiSdkToolSet({ read_thing: descriptor });
|
||||
const output = await tools.read_thing.execute?.({ id: 'a' }, { toolCallId: 'call-1', messages: [] } as never);
|
||||
const modelOutput = tools.read_thing.toModelOutput?.({ output } as never);
|
||||
|
||||
expect(modelOutput).toEqual({ type: 'text', value: 'Read a' });
|
||||
});
|
||||
|
||||
it('builds Claude SDK tools that return text content only', async () => {
|
||||
const tools = createClaudeSdkTools({ read_thing: descriptor });
|
||||
const result = await tools[0].handler({ id: 'b' } as never, {});
|
||||
|
||||
expect(result).toEqual({ content: [{ type: 'text', text: 'Read b' }] });
|
||||
});
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue