feat(llm): add local AI SDK DevTools tracing

.gitignore

@@ -43,6 +43,7 @@ yarn-error.log*
 
 # Local project runtime state
 .ktx/
+**/.devtools/
 *.db
 *.sqlite
 *.sqlite3

README.md

@@ -159,6 +159,29 @@ pnpm run link:dev
 ktx-dev --help
 ```
 
+### Debug LLM traces
+
+KTX can capture local AI SDK DevTools traces for LLM calls that run through the
+KTX provider. Enable it with an environment flag when running an LLM-backed
+command:
+
+```bash
+KTX_AI_DEVTOOLS_ENABLED=true ktx dev ingest run \
+  --connection-id warehouse \
+  --adapter metabase
+```
+
+Traces are written to `.devtools/generations.json` under the current working
+directory. To inspect them, run:
+
+```bash
+pnpm dlx @ai-sdk/devtools
+```
+
+Then open `http://localhost:4983`. These traces are local-development-only and
+store prompts, model outputs, tool arguments/results, and raw provider payloads
+in plain text. Do not enable this in production or for sensitive runs.
+
 The repository uses `pnpm` for TypeScript packages and `uv` for Python
 packages. See [Contributing](docs-site/content/docs/community/contributing.mdx)
 for full development setup, testing, and PR guidelines.

packages/llm/package.json

@@ -27,6 +27,7 @@
   },
   "dependencies": {
     "@ai-sdk/anthropic": "3.0.71",
+    "@ai-sdk/devtools": "0.0.17",
     "@ai-sdk/google-vertex": "^4.0.112",
     "ai": "^6.0.168",
     "openai": "^6.25.0"

packages/llm/src/model-health.test.ts

@@ -1,3 +1,4 @@
+import { wrapLanguageModel as defaultWrapLanguageModel } from 'ai';
 import { describe, expect, it, vi } from 'vitest';
 import { runKtxLlmHealthCheck } from './model-health.js';
 
@@ -7,6 +8,7 @@ describe('KTX LLM health check', () => {
   it('runs a minimal non-streaming model call through the configured provider', async () => {
     const generateText = vi.fn(async () => ({ text: 'ok' }));
     const createAnthropic = vi.fn(() => vi.fn(() => anthropicModel));
+    const wrapLanguageModel = vi.fn(defaultWrapLanguageModel);
 
     await expect(
       runKtxLlmHealthCheck(
@@ -15,7 +17,7 @@ describe('KTX LLM health check', () => {
           anthropic: { apiKey: 'sk-ant-test' },
           modelSlots: { default: 'claude-sonnet-4-6' },
         },
-        { deps: { createAnthropic, generateText } },
+        { deps: { createAnthropic, generateText, devtoolsEnabled: true, wrapLanguageModel } },
       ),
     ).resolves.toEqual({ ok: true });
 
@@ -32,6 +34,7 @@ describe('KTX LLM health check', () => {
         maxOutputTokens: 8,
       }),
     );
+    expect(wrapLanguageModel).not.toHaveBeenCalled();
   });
 
   it('returns a failed result without exposing secret values', async () => {

packages/llm/src/model-health.ts

@@ -41,7 +41,7 @@ export async function runKtxLlmHealthCheck(
 ): Promise<KtxLlmHealthCheckResult> {
   try {
     const { generateText: runGenerateTextOverride, ...providerDeps } = options.deps ?? {};
-    const provider = createKtxLlmProvider(config, providerDeps);
+    const provider = createKtxLlmProvider(config, { ...providerDeps, devtoolsEnabled: false });
     const runGenerateText = runGenerateTextOverride ?? generateText;
     await withTimeout(
       runGenerateText({

packages/llm/src/model-provider.test.ts

@@ -1,10 +1,138 @@
-import type { LanguageModel } from 'ai';
+import { devToolsMiddleware as defaultDevToolsMiddleware } from '@ai-sdk/devtools';
+import { wrapLanguageModel as defaultWrapLanguageModel, type LanguageModel } from 'ai';
 import { describe, expect, it, vi } from 'vitest';
-import { createKtxLlmProvider } from './model-provider.js';
+import { createKtxLlmProvider, type KtxLlmProviderFactoryDeps } from './model-provider.js';
 
 const languageModel = (modelId: string, provider = 'test'): LanguageModel => ({ modelId, provider }) as LanguageModel;
+const devtoolsMiddleware = (): ReturnType<typeof defaultDevToolsMiddleware> => ({ specificationVersion: 'v3' });
+const wrapWith = (model: LanguageModel) =>
+  vi.fn((_options: Parameters<typeof defaultWrapLanguageModel>[0]) => model as ReturnType<typeof defaultWrapLanguageModel>);
 
 describe('createKtxLlmProvider', () => {
+  it('wraps language models with DevTools middleware when explicitly enabled', () => {
+    const anthropicModel = languageModel('claude-sonnet-4-6', 'anthropic');
+    const wrappedModel = languageModel('claude-sonnet-4-6', 'anthropic-devtools');
+    const middleware = devtoolsMiddleware();
+    const wrapLanguageModel = wrapWith(wrappedModel);
+    const devToolsMiddleware = vi.fn(devtoolsMiddleware);
+
+    const provider = createKtxLlmProvider(
+      {
+        backend: 'anthropic',
+        anthropic: { apiKey: 'test-anthropic-key' }, // pragma: allowlist secret
+        modelSlots: { default: 'claude-sonnet-4-6' },
+        promptCaching: { enabled: false },
+      },
+      {
+        createAnthropic: vi.fn(() => vi.fn(() => anthropicModel)),
+        devtoolsEnabled: true,
+        wrapLanguageModel,
+        devToolsMiddleware,
+      } satisfies KtxLlmProviderFactoryDeps,
+    );
+
+    expect(provider.getModel('default')).toBe(wrappedModel);
+    expect(devToolsMiddleware).toHaveBeenCalledTimes(1);
+    expect(wrapLanguageModel).toHaveBeenCalledWith({
+      model: anthropicModel,
+      middleware,
+      modelId: 'claude-sonnet-4-6',
+      providerId: 'anthropic',
+    });
+  });
+
+  it('does not wrap language models by default', () => {
+    const anthropicModel = languageModel('claude-sonnet-4-6', 'anthropic');
+    const wrapLanguageModel = vi.fn(defaultWrapLanguageModel);
+    const devToolsMiddleware = vi.fn(defaultDevToolsMiddleware);
+
+    const provider = createKtxLlmProvider(
+      {
+        backend: 'anthropic',
+        anthropic: { apiKey: 'test-anthropic-key' }, // pragma: allowlist secret
+        modelSlots: { default: 'claude-sonnet-4-6' },
+        promptCaching: { enabled: false },
+      },
+      {
+        createAnthropic: vi.fn(() => vi.fn(() => anthropicModel)),
+        wrapLanguageModel,
+        devToolsMiddleware,
+      } satisfies KtxLlmProviderFactoryDeps,
+    );
+
+    expect(provider.getModel('default')).toBe(anthropicModel);
+    expect(wrapLanguageModel).not.toHaveBeenCalled();
+    expect(devToolsMiddleware).not.toHaveBeenCalled();
+  });
+
+  it('wraps language models when KTX_AI_DEVTOOLS_ENABLED is true', () => {
+    const originalEnv = process.env.KTX_AI_DEVTOOLS_ENABLED;
+    process.env.KTX_AI_DEVTOOLS_ENABLED = 'true';
+    try {
+      const gatewayModel = languageModel('anthropic/claude-sonnet-4-6', 'gateway');
+      const wrappedModel = languageModel('anthropic/claude-sonnet-4-6', 'gateway-devtools');
+      const wrapLanguageModel = wrapWith(wrappedModel);
+
+      const provider = createKtxLlmProvider(
+        {
+          backend: 'gateway',
+          gateway: { baseURL: 'https://gateway.test/v1' },
+          modelSlots: { default: 'anthropic/claude-sonnet-4-6' },
+          promptCaching: { enabled: false },
+        },
+        {
+          createGateway: vi.fn(() => vi.fn(() => gatewayModel)),
+          wrapLanguageModel,
+          devToolsMiddleware: vi.fn(devtoolsMiddleware),
+        } satisfies KtxLlmProviderFactoryDeps,
+      );
+
+      expect(provider.getModel('default')).toBe(wrappedModel);
+      expect(wrapLanguageModel).toHaveBeenCalledTimes(1);
+    } finally {
+      if (originalEnv === undefined) {
+        delete process.env.KTX_AI_DEVTOOLS_ENABLED;
+      } else {
+        process.env.KTX_AI_DEVTOOLS_ENABLED = originalEnv;
+      }
+    }
+  });
+
+  it('does not wrap language models in production even when enabled', () => {
+    const originalNodeEnv = process.env.NODE_ENV;
+    process.env.NODE_ENV = 'production';
+    try {
+      const anthropicModel = languageModel('claude-sonnet-4-6', 'anthropic');
+      const wrapLanguageModel = vi.fn(defaultWrapLanguageModel);
+      const devToolsMiddleware = vi.fn(defaultDevToolsMiddleware);
+
+      const provider = createKtxLlmProvider(
+        {
+          backend: 'anthropic',
+          anthropic: { apiKey: 'test-anthropic-key' }, // pragma: allowlist secret
+          modelSlots: { default: 'claude-sonnet-4-6' },
+          promptCaching: { enabled: false },
+        },
+        {
+          createAnthropic: vi.fn(() => vi.fn(() => anthropicModel)),
+          devtoolsEnabled: true,
+          wrapLanguageModel,
+          devToolsMiddleware,
+        } satisfies KtxLlmProviderFactoryDeps,
+      );
+
+      expect(provider.getModel('default')).toBe(anthropicModel);
+      expect(wrapLanguageModel).not.toHaveBeenCalled();
+      expect(devToolsMiddleware).not.toHaveBeenCalled();
+    } finally {
+      if (originalNodeEnv === undefined) {
+        delete process.env.NODE_ENV;
+      } else {
+        process.env.NODE_ENV = originalNodeEnv;
+      }
+    }
+  });
+
   it('uses direct Anthropic with both beta headers', () => {
     const anthropicModel = languageModel('claude-sonnet-4-6', 'anthropic');
     const anthropic = vi.fn(() => anthropicModel);

packages/llm/src/model-provider.ts

@@ -1,6 +1,7 @@
 import { createAnthropic } from '@ai-sdk/anthropic';
+import { devToolsMiddleware } from '@ai-sdk/devtools';
 import { createVertexAnthropic } from '@ai-sdk/google-vertex/anthropic';
-import { createGateway, generateText, type LanguageModel } from 'ai';
+import { createGateway, generateText, wrapLanguageModel, type LanguageModel } from 'ai';
 import { createKtxToolCallRepairHandler } from './repair.js';
 import type {
   KtxLlmConfig,
@@ -21,6 +22,9 @@ export interface KtxLlmProviderFactoryDeps {
   createVertexAnthropic?: VertexAnthropicFactory;
   createGateway?: GatewayFactory;
   generateText?: typeof generateText;
+  devtoolsEnabled?: boolean;
+  wrapLanguageModel?: typeof wrapLanguageModel;
+  devToolsMiddleware?: typeof devToolsMiddleware;
 }
 
 const DEFAULT_PROMPT_CACHING: KtxPromptCachingConfig = {
@@ -40,10 +44,27 @@ function resolvePromptCaching(config: KtxLlmConfig): KtxPromptCachingConfig {
   return { ...DEFAULT_PROMPT_CACHING, ...config.promptCaching };
 }
 
+function resolveDevtoolsEnabled(override: boolean | undefined): boolean {
+  if (process.env.NODE_ENV === 'production') {
+    return false;
+  }
+  if (override !== undefined) {
+    return override;
+  }
+  const value = process.env.KTX_AI_DEVTOOLS_ENABLED?.trim().toLowerCase();
+  return value === 'true' || value === '1' || value === 'yes';
+}
+
 export function modelIdFromLanguageModel(model: LanguageModel | string): string {
   return typeof model === 'string' ? model : ((model as { modelId?: string }).modelId ?? '');
 }
 
+function providerIdFromLanguageModel(model: Exclude<LanguageModel, string>): string | undefined {
+  return typeof (model as { provider?: unknown }).provider === 'string'
+    ? (model as { provider: string }).provider
+    : undefined;
+}
+
 export function isAnthropicProtocolModel(model: LanguageModel | string): boolean {
   const modelId = modelIdFromLanguageModel(model);
   return modelId.startsWith('claude-') || modelId.startsWith('anthropic/') || modelId.includes('/claude-');
@@ -53,6 +74,9 @@ class DefaultKtxLlmProvider implements KtxLlmProvider {
   private readonly promptCaching: KtxPromptCachingConfig;
   private readonly getModelByResolvedName: (modelId: string) => LanguageModel;
   private readonly runGenerateText: typeof generateText;
+  private readonly devtoolsEnabled: boolean;
+  private readonly runWrapLanguageModel: typeof wrapLanguageModel;
+  private readonly createDevToolsMiddleware: typeof devToolsMiddleware;
 
   constructor(
     private readonly config: KtxLlmConfig,
@@ -60,6 +84,9 @@ class DefaultKtxLlmProvider implements KtxLlmProvider {
   ) {
     this.promptCaching = resolvePromptCaching(config);
     this.runGenerateText = deps.generateText ?? generateText;
+    this.devtoolsEnabled = resolveDevtoolsEnabled(deps.devtoolsEnabled);
+    this.runWrapLanguageModel = deps.wrapLanguageModel ?? wrapLanguageModel;
+    this.createDevToolsMiddleware = deps.devToolsMiddleware ?? devToolsMiddleware;
     this.getModelByResolvedName = this.createModelFactory(config, deps);
   }
 
@@ -68,7 +95,7 @@ class DefaultKtxLlmProvider implements KtxLlmProvider {
   }
 
   getModelByName(modelId: string): LanguageModel {
-    return this.getModelByResolvedName(modelId);
+    return this.withDevtools(this.getModelByResolvedName(modelId));
   }
 
   cacheMarker(ttl: KtxPromptCacheTtl, model?: LanguageModel | string) {
@@ -113,6 +140,18 @@ class DefaultKtxLlmProvider implements KtxLlmProvider {
     return this.config.modelSlots[role] ?? this.config.modelSlots.default;
   }
 
+  private withDevtools(model: LanguageModel): LanguageModel {
+    if (!this.devtoolsEnabled || typeof model === 'string') {
+      return model;
+    }
+    return this.runWrapLanguageModel({
+      model: model as Parameters<typeof wrapLanguageModel>[0]['model'],
+      middleware: this.createDevToolsMiddleware(),
+      modelId: modelIdFromLanguageModel(model),
+      providerId: providerIdFromLanguageModel(model),
+    });
+  }
+
   private createModelFactory(config: KtxLlmConfig, deps: KtxLlmProviderFactoryDeps): (modelId: string) => LanguageModel {
     if (config.backend === 'anthropic') {
       const anthropic = (deps.createAnthropic ?? createAnthropic)({

pnpm-lock.yaml

@@ -355,6 +355,9 @@ importers:
       '@ai-sdk/anthropic':
         specifier: 3.0.71
         version: 3.0.71(zod@4.4.3)
+      '@ai-sdk/devtools':
+        specifier: 0.0.17
+        version: 0.0.17
       '@ai-sdk/google-vertex':
         specifier: ^4.0.112
         version: 4.0.118(zod@4.4.3)
@@ -389,6 +392,11 @@ packages:
     peerDependencies:
       zod: ^3.25.76 || ^4.1.8
 
+  '@ai-sdk/devtools@0.0.17':
+    resolution: {integrity: sha512-CJgo+3DMHOJbxxq1qTgnW4vpFXgBW1pHePMimBW4Go5FPU7iLqppoGX/UC798IXqlD3hncQRPfyBLZjbsJC91w==}
+    engines: {node: '>=18'}
+    hasBin: true
+
   '@ai-sdk/gateway@3.0.104':
     resolution: {integrity: sha512-ZKX5n74io8VIRlhIMSLWVlvT3sXC8Z7cZ9GHuWBWZDVi96+62AIsWuLGvMfcBA1STYuSoDrp6rIziZmvrTq0TA==}
     engines: {node: '>=18'}
@@ -4576,6 +4584,12 @@ snapshots:
       '@ai-sdk/provider-utils': 4.0.26(zod@4.4.3)
       zod: 4.4.3
 
+  '@ai-sdk/devtools@0.0.17':
+    dependencies:
+      '@ai-sdk/provider': 3.0.10
+      '@hono/node-server': 1.19.14(hono@4.12.15)
+      hono: 4.12.15
+
   '@ai-sdk/gateway@3.0.104(zod@4.3.6)':
     dependencies:
       '@ai-sdk/provider': 3.0.8