diff --git a/.gitignore b/.gitignore index eb63517a..ed14196b 100644 --- a/.gitignore +++ b/.gitignore @@ -43,6 +43,7 @@ yarn-error.log* # Local project runtime state .ktx/ +**/.devtools/ *.db *.sqlite *.sqlite3 diff --git a/README.md b/README.md index 84592226..255aa51b 100644 --- a/README.md +++ b/README.md @@ -159,6 +159,29 @@ pnpm run link:dev ktx-dev --help ``` +### Debug LLM traces + +KTX can capture local AI SDK DevTools traces for LLM calls that run through the +KTX provider. Enable it with an environment flag when running an LLM-backed +command: + +```bash +KTX_AI_DEVTOOLS_ENABLED=true ktx dev ingest run \ + --connection-id warehouse \ + --adapter metabase +``` + +Traces are written to `.devtools/generations.json` under the current working +directory. To inspect them, run: + +```bash +pnpm dlx @ai-sdk/devtools +``` + +Then open `http://localhost:4983`. These traces are local-development-only and +store prompts, model outputs, tool arguments/results, and raw provider payloads +in plain text. Do not enable this in production or for sensitive runs. + The repository uses `pnpm` for TypeScript packages and `uv` for Python packages. See [Contributing](docs-site/content/docs/community/contributing.mdx) for full development setup, testing, and PR guidelines. diff --git a/packages/llm/package.json b/packages/llm/package.json index fc7deeba..13f49666 100644 --- a/packages/llm/package.json +++ b/packages/llm/package.json @@ -27,6 +27,7 @@ }, "dependencies": { "@ai-sdk/anthropic": "3.0.71", + "@ai-sdk/devtools": "0.0.17", "@ai-sdk/google-vertex": "^4.0.112", "ai": "^6.0.168", "openai": "^6.25.0" diff --git a/packages/llm/src/model-health.test.ts b/packages/llm/src/model-health.test.ts index 003c12d5..d1b3df47 100644 --- a/packages/llm/src/model-health.test.ts +++ b/packages/llm/src/model-health.test.ts @@ -1,3 +1,4 @@ +import { wrapLanguageModel as defaultWrapLanguageModel } from 'ai'; import { describe, expect, it, vi } from 'vitest'; import { runKtxLlmHealthCheck } from './model-health.js'; @@ -7,6 +8,7 @@ describe('KTX LLM health check', () => { it('runs a minimal non-streaming model call through the configured provider', async () => { const generateText = vi.fn(async () => ({ text: 'ok' })); const createAnthropic = vi.fn(() => vi.fn(() => anthropicModel)); + const wrapLanguageModel = vi.fn(defaultWrapLanguageModel); await expect( runKtxLlmHealthCheck( @@ -15,7 +17,7 @@ describe('KTX LLM health check', () => { anthropic: { apiKey: 'sk-ant-test' }, modelSlots: { default: 'claude-sonnet-4-6' }, }, - { deps: { createAnthropic, generateText } }, + { deps: { createAnthropic, generateText, devtoolsEnabled: true, wrapLanguageModel } }, ), ).resolves.toEqual({ ok: true }); @@ -32,6 +34,7 @@ describe('KTX LLM health check', () => { maxOutputTokens: 8, }), ); + expect(wrapLanguageModel).not.toHaveBeenCalled(); }); it('returns a failed result without exposing secret values', async () => { diff --git a/packages/llm/src/model-health.ts b/packages/llm/src/model-health.ts index 131822b6..abbc2735 100644 --- a/packages/llm/src/model-health.ts +++ b/packages/llm/src/model-health.ts @@ -41,7 +41,7 @@ export async function runKtxLlmHealthCheck( ): Promise { try { const { generateText: runGenerateTextOverride, ...providerDeps } = options.deps ?? {}; - const provider = createKtxLlmProvider(config, providerDeps); + const provider = createKtxLlmProvider(config, { ...providerDeps, devtoolsEnabled: false }); const runGenerateText = runGenerateTextOverride ?? generateText; await withTimeout( runGenerateText({ diff --git a/packages/llm/src/model-provider.test.ts b/packages/llm/src/model-provider.test.ts index 55dd5da9..ff65a12a 100644 --- a/packages/llm/src/model-provider.test.ts +++ b/packages/llm/src/model-provider.test.ts @@ -1,10 +1,138 @@ -import type { LanguageModel } from 'ai'; +import { devToolsMiddleware as defaultDevToolsMiddleware } from '@ai-sdk/devtools'; +import { wrapLanguageModel as defaultWrapLanguageModel, type LanguageModel } from 'ai'; import { describe, expect, it, vi } from 'vitest'; -import { createKtxLlmProvider } from './model-provider.js'; +import { createKtxLlmProvider, type KtxLlmProviderFactoryDeps } from './model-provider.js'; const languageModel = (modelId: string, provider = 'test'): LanguageModel => ({ modelId, provider }) as LanguageModel; +const devtoolsMiddleware = (): ReturnType => ({ specificationVersion: 'v3' }); +const wrapWith = (model: LanguageModel) => + vi.fn((_options: Parameters[0]) => model as ReturnType); describe('createKtxLlmProvider', () => { + it('wraps language models with DevTools middleware when explicitly enabled', () => { + const anthropicModel = languageModel('claude-sonnet-4-6', 'anthropic'); + const wrappedModel = languageModel('claude-sonnet-4-6', 'anthropic-devtools'); + const middleware = devtoolsMiddleware(); + const wrapLanguageModel = wrapWith(wrappedModel); + const devToolsMiddleware = vi.fn(devtoolsMiddleware); + + const provider = createKtxLlmProvider( + { + backend: 'anthropic', + anthropic: { apiKey: 'test-anthropic-key' }, // pragma: allowlist secret + modelSlots: { default: 'claude-sonnet-4-6' }, + promptCaching: { enabled: false }, + }, + { + createAnthropic: vi.fn(() => vi.fn(() => anthropicModel)), + devtoolsEnabled: true, + wrapLanguageModel, + devToolsMiddleware, + } satisfies KtxLlmProviderFactoryDeps, + ); + + expect(provider.getModel('default')).toBe(wrappedModel); + expect(devToolsMiddleware).toHaveBeenCalledTimes(1); + expect(wrapLanguageModel).toHaveBeenCalledWith({ + model: anthropicModel, + middleware, + modelId: 'claude-sonnet-4-6', + providerId: 'anthropic', + }); + }); + + it('does not wrap language models by default', () => { + const anthropicModel = languageModel('claude-sonnet-4-6', 'anthropic'); + const wrapLanguageModel = vi.fn(defaultWrapLanguageModel); + const devToolsMiddleware = vi.fn(defaultDevToolsMiddleware); + + const provider = createKtxLlmProvider( + { + backend: 'anthropic', + anthropic: { apiKey: 'test-anthropic-key' }, // pragma: allowlist secret + modelSlots: { default: 'claude-sonnet-4-6' }, + promptCaching: { enabled: false }, + }, + { + createAnthropic: vi.fn(() => vi.fn(() => anthropicModel)), + wrapLanguageModel, + devToolsMiddleware, + } satisfies KtxLlmProviderFactoryDeps, + ); + + expect(provider.getModel('default')).toBe(anthropicModel); + expect(wrapLanguageModel).not.toHaveBeenCalled(); + expect(devToolsMiddleware).not.toHaveBeenCalled(); + }); + + it('wraps language models when KTX_AI_DEVTOOLS_ENABLED is true', () => { + const originalEnv = process.env.KTX_AI_DEVTOOLS_ENABLED; + process.env.KTX_AI_DEVTOOLS_ENABLED = 'true'; + try { + const gatewayModel = languageModel('anthropic/claude-sonnet-4-6', 'gateway'); + const wrappedModel = languageModel('anthropic/claude-sonnet-4-6', 'gateway-devtools'); + const wrapLanguageModel = wrapWith(wrappedModel); + + const provider = createKtxLlmProvider( + { + backend: 'gateway', + gateway: { baseURL: 'https://gateway.test/v1' }, + modelSlots: { default: 'anthropic/claude-sonnet-4-6' }, + promptCaching: { enabled: false }, + }, + { + createGateway: vi.fn(() => vi.fn(() => gatewayModel)), + wrapLanguageModel, + devToolsMiddleware: vi.fn(devtoolsMiddleware), + } satisfies KtxLlmProviderFactoryDeps, + ); + + expect(provider.getModel('default')).toBe(wrappedModel); + expect(wrapLanguageModel).toHaveBeenCalledTimes(1); + } finally { + if (originalEnv === undefined) { + delete process.env.KTX_AI_DEVTOOLS_ENABLED; + } else { + process.env.KTX_AI_DEVTOOLS_ENABLED = originalEnv; + } + } + }); + + it('does not wrap language models in production even when enabled', () => { + const originalNodeEnv = process.env.NODE_ENV; + process.env.NODE_ENV = 'production'; + try { + const anthropicModel = languageModel('claude-sonnet-4-6', 'anthropic'); + const wrapLanguageModel = vi.fn(defaultWrapLanguageModel); + const devToolsMiddleware = vi.fn(defaultDevToolsMiddleware); + + const provider = createKtxLlmProvider( + { + backend: 'anthropic', + anthropic: { apiKey: 'test-anthropic-key' }, // pragma: allowlist secret + modelSlots: { default: 'claude-sonnet-4-6' }, + promptCaching: { enabled: false }, + }, + { + createAnthropic: vi.fn(() => vi.fn(() => anthropicModel)), + devtoolsEnabled: true, + wrapLanguageModel, + devToolsMiddleware, + } satisfies KtxLlmProviderFactoryDeps, + ); + + expect(provider.getModel('default')).toBe(anthropicModel); + expect(wrapLanguageModel).not.toHaveBeenCalled(); + expect(devToolsMiddleware).not.toHaveBeenCalled(); + } finally { + if (originalNodeEnv === undefined) { + delete process.env.NODE_ENV; + } else { + process.env.NODE_ENV = originalNodeEnv; + } + } + }); + it('uses direct Anthropic with both beta headers', () => { const anthropicModel = languageModel('claude-sonnet-4-6', 'anthropic'); const anthropic = vi.fn(() => anthropicModel); diff --git a/packages/llm/src/model-provider.ts b/packages/llm/src/model-provider.ts index 66a9fddd..6dbdcb06 100644 --- a/packages/llm/src/model-provider.ts +++ b/packages/llm/src/model-provider.ts @@ -1,6 +1,7 @@ import { createAnthropic } from '@ai-sdk/anthropic'; +import { devToolsMiddleware } from '@ai-sdk/devtools'; import { createVertexAnthropic } from '@ai-sdk/google-vertex/anthropic'; -import { createGateway, generateText, type LanguageModel } from 'ai'; +import { createGateway, generateText, wrapLanguageModel, type LanguageModel } from 'ai'; import { createKtxToolCallRepairHandler } from './repair.js'; import type { KtxLlmConfig, @@ -21,6 +22,9 @@ export interface KtxLlmProviderFactoryDeps { createVertexAnthropic?: VertexAnthropicFactory; createGateway?: GatewayFactory; generateText?: typeof generateText; + devtoolsEnabled?: boolean; + wrapLanguageModel?: typeof wrapLanguageModel; + devToolsMiddleware?: typeof devToolsMiddleware; } const DEFAULT_PROMPT_CACHING: KtxPromptCachingConfig = { @@ -40,10 +44,27 @@ function resolvePromptCaching(config: KtxLlmConfig): KtxPromptCachingConfig { return { ...DEFAULT_PROMPT_CACHING, ...config.promptCaching }; } +function resolveDevtoolsEnabled(override: boolean | undefined): boolean { + if (process.env.NODE_ENV === 'production') { + return false; + } + if (override !== undefined) { + return override; + } + const value = process.env.KTX_AI_DEVTOOLS_ENABLED?.trim().toLowerCase(); + return value === 'true' || value === '1' || value === 'yes'; +} + export function modelIdFromLanguageModel(model: LanguageModel | string): string { return typeof model === 'string' ? model : ((model as { modelId?: string }).modelId ?? ''); } +function providerIdFromLanguageModel(model: Exclude): string | undefined { + return typeof (model as { provider?: unknown }).provider === 'string' + ? (model as { provider: string }).provider + : undefined; +} + export function isAnthropicProtocolModel(model: LanguageModel | string): boolean { const modelId = modelIdFromLanguageModel(model); return modelId.startsWith('claude-') || modelId.startsWith('anthropic/') || modelId.includes('/claude-'); @@ -53,6 +74,9 @@ class DefaultKtxLlmProvider implements KtxLlmProvider { private readonly promptCaching: KtxPromptCachingConfig; private readonly getModelByResolvedName: (modelId: string) => LanguageModel; private readonly runGenerateText: typeof generateText; + private readonly devtoolsEnabled: boolean; + private readonly runWrapLanguageModel: typeof wrapLanguageModel; + private readonly createDevToolsMiddleware: typeof devToolsMiddleware; constructor( private readonly config: KtxLlmConfig, @@ -60,6 +84,9 @@ class DefaultKtxLlmProvider implements KtxLlmProvider { ) { this.promptCaching = resolvePromptCaching(config); this.runGenerateText = deps.generateText ?? generateText; + this.devtoolsEnabled = resolveDevtoolsEnabled(deps.devtoolsEnabled); + this.runWrapLanguageModel = deps.wrapLanguageModel ?? wrapLanguageModel; + this.createDevToolsMiddleware = deps.devToolsMiddleware ?? devToolsMiddleware; this.getModelByResolvedName = this.createModelFactory(config, deps); } @@ -68,7 +95,7 @@ class DefaultKtxLlmProvider implements KtxLlmProvider { } getModelByName(modelId: string): LanguageModel { - return this.getModelByResolvedName(modelId); + return this.withDevtools(this.getModelByResolvedName(modelId)); } cacheMarker(ttl: KtxPromptCacheTtl, model?: LanguageModel | string) { @@ -113,6 +140,18 @@ class DefaultKtxLlmProvider implements KtxLlmProvider { return this.config.modelSlots[role] ?? this.config.modelSlots.default; } + private withDevtools(model: LanguageModel): LanguageModel { + if (!this.devtoolsEnabled || typeof model === 'string') { + return model; + } + return this.runWrapLanguageModel({ + model: model as Parameters[0]['model'], + middleware: this.createDevToolsMiddleware(), + modelId: modelIdFromLanguageModel(model), + providerId: providerIdFromLanguageModel(model), + }); + } + private createModelFactory(config: KtxLlmConfig, deps: KtxLlmProviderFactoryDeps): (modelId: string) => LanguageModel { if (config.backend === 'anthropic') { const anthropic = (deps.createAnthropic ?? createAnthropic)({ diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e74389f9..0cd6dbeb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -355,6 +355,9 @@ importers: '@ai-sdk/anthropic': specifier: 3.0.71 version: 3.0.71(zod@4.4.3) + '@ai-sdk/devtools': + specifier: 0.0.17 + version: 0.0.17 '@ai-sdk/google-vertex': specifier: ^4.0.112 version: 4.0.118(zod@4.4.3) @@ -389,6 +392,11 @@ packages: peerDependencies: zod: ^3.25.76 || ^4.1.8 + '@ai-sdk/devtools@0.0.17': + resolution: {integrity: sha512-CJgo+3DMHOJbxxq1qTgnW4vpFXgBW1pHePMimBW4Go5FPU7iLqppoGX/UC798IXqlD3hncQRPfyBLZjbsJC91w==} + engines: {node: '>=18'} + hasBin: true + '@ai-sdk/gateway@3.0.104': resolution: {integrity: sha512-ZKX5n74io8VIRlhIMSLWVlvT3sXC8Z7cZ9GHuWBWZDVi96+62AIsWuLGvMfcBA1STYuSoDrp6rIziZmvrTq0TA==} engines: {node: '>=18'} @@ -4576,6 +4584,12 @@ snapshots: '@ai-sdk/provider-utils': 4.0.26(zod@4.4.3) zod: 4.4.3 + '@ai-sdk/devtools@0.0.17': + dependencies: + '@ai-sdk/provider': 3.0.10 + '@hono/node-server': 1.19.14(hono@4.12.15) + hono: 4.12.15 + '@ai-sdk/gateway@3.0.104(zod@4.3.6)': dependencies: '@ai-sdk/provider': 3.0.8