mirror of
https://github.com/Kaelio/ktx.git
synced 2026-06-07 07:55:13 +02:00
Compare commits
No commits in common. "main" and "v0.9.0" have entirely different histories.
123 changed files with 352 additions and 7774 deletions
21
AGENTS.md
21
AGENTS.md
|
|
@ -337,8 +337,7 @@ use `PascalCase` without the suffix.
|
|||
|
||||
## Telemetry
|
||||
|
||||
**ktx** ships PostHog usage telemetry. Catalog telemetry events use strict
|
||||
schemas. When adding commands or events:
|
||||
**ktx** ships PostHog usage telemetry. When adding commands or events:
|
||||
|
||||
- **MUST NOT**: Add fields that carry user data — file paths, hostnames,
|
||||
environment values, SQL text, schema/table/column names, error messages,
|
||||
|
|
@ -355,24 +354,6 @@ schemas. When adding commands or events:
|
|||
of collected data changes. Adding another event with no new field types
|
||||
needs no docs change.
|
||||
|
||||
### Error reports
|
||||
|
||||
**ktx** also sends PostHog Error Tracking `$exception` events when telemetry is
|
||||
enabled. This channel is separate from the strict catalog event schema and is
|
||||
used only for exception diagnostics.
|
||||
|
||||
`$exception` events may include stack frames, error class names, raw error
|
||||
messages, cause chains, `source`, `handled`, `fatal`, runtime version fields,
|
||||
OS/runtime fields, and the hashed `projectId` when known. Stack frames may
|
||||
include local file paths and the local username when those appear in paths.
|
||||
|
||||
`$exception` events must never intentionally include secrets, credentials,
|
||||
database URLs, auth headers, raw argv, raw environment values, SQL text,
|
||||
schema/table/column names as explicit properties, customer row data, user prompt
|
||||
text, or raw MCP arguments. Reporters must redact call-site-provided secret
|
||||
snapshots and common static credential patterns before the SDK serializes the
|
||||
exception.
|
||||
|
||||
## Documentation and Specs
|
||||
|
||||
- Keep public documentation in `README.md`, package READMEs, example READMEs,
|
||||
|
|
|
|||
22
README.md
22
README.md
|
|
@ -13,7 +13,7 @@
|
|||
<a href="https://docs.kaelio.com/ktx/docs/"><img src="https://img.shields.io/badge/docs-ktx-22c55e?style=flat-square" alt="Documentation" /></a>
|
||||
<a href="https://join.slack.com/t/ktxcommunity/shared_invite/zt-3y9b44m1x-LVyNNJD5nwaZHq4XS29LMQ"><img src="https://img.shields.io/badge/slack-join%20community-4A154B?style=flat-square&logo=slack&logoColor=white" alt="Join the ktx Slack community" /></a>
|
||||
<a href="https://github.com/Kaelio/ktx/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue?style=flat-square" alt="License" /></a>
|
||||
<a href="https://www.ycombinator.com/companies/kaelio"><img src="https://img.shields.io/badge/Y%20Combinator-P25-orange?style=flat-square" alt="Y Combinator P25" /></a>
|
||||
<a href="https://www.ycombinator.com/companies?batch=P25"><img src="https://img.shields.io/badge/Y%20Combinator-P25-orange?style=flat-square" alt="Y Combinator P25" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
|
|
@ -23,10 +23,6 @@
|
|||
<a href="https://join.slack.com/t/ktxcommunity/shared_invite/zt-3y9b44m1x-LVyNNJD5nwaZHq4XS29LMQ"><b>Slack</b></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<sub>Built and maintained by <a href="https://www.kaelio.com"><b>Kaelio</b></a></sub>
|
||||
</p>
|
||||
|
||||
---
|
||||
|
||||
**ktx** is a self-improving context layer that teaches agents how to query your
|
||||
|
|
@ -251,17 +247,11 @@ uv run pytest -q
|
|||
|
||||
## Telemetry
|
||||
|
||||
**ktx** collects privacy-conscious usage telemetry to understand installs and
|
||||
improve setup, command reliability, and data-agent workflows. Catalog telemetry
|
||||
events do not record file paths, hostnames, SQL, schema names, table names,
|
||||
column names, error messages, raw environment values, or argv. Error reports use
|
||||
PostHog Error Tracking and can include stack frames and raw error messages,
|
||||
which may contain local file paths or the local username in those paths.
|
||||
**ktx** redacts secrets, credentials, database URLs, auth headers, argv, raw
|
||||
environment values, SQL text, row data, and user-typed prompt or MCP argument
|
||||
text from the explicit `$exception` payload. See
|
||||
[Telemetry](https://docs.kaelio.com/ktx/docs/community/telemetry) for the event
|
||||
catalog and opt-out options.
|
||||
**ktx** collects anonymous usage telemetry from interactive CLI runs to
|
||||
improve setup, command reliability, and data-agent workflows. No file paths,
|
||||
hostnames, SQL, schema names, error messages, or argv are recorded. See
|
||||
[Telemetry](https://docs.kaelio.com/ktx/docs/community/telemetry) for the
|
||||
event catalog and opt-out options.
|
||||
|
||||
## License
|
||||
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
Before Width: | Height: | Size: 63 KiB After Width: | Height: | Size: 63 KiB |
|
|
@ -5,7 +5,7 @@ import { SlackIcon } from "@/components/slack-icon";
|
|||
|
||||
export const baseOptions: BaseLayoutProps = {
|
||||
nav: {
|
||||
title: Logo,
|
||||
title: <Logo />,
|
||||
transparentMode: "top",
|
||||
},
|
||||
links: [
|
||||
|
|
|
|||
|
|
@ -305,8 +305,8 @@ export const runtimeEdges: Edge[] = [
|
|||
sourceHandle: "to-context",
|
||||
target: "context",
|
||||
targetHandle: "in",
|
||||
type: "smoothstep",
|
||||
label: "search + read",
|
||||
type: "default",
|
||||
label: "search",
|
||||
...labelBg,
|
||||
style: edgeStyle,
|
||||
markerStart: marker,
|
||||
|
|
@ -318,7 +318,7 @@ export const runtimeEdges: Edge[] = [
|
|||
sourceHandle: "to-warehouse",
|
||||
target: "warehouse",
|
||||
targetHandle: "in",
|
||||
type: "smoothstep",
|
||||
type: "default",
|
||||
label: "read-only",
|
||||
...labelBg,
|
||||
style: edgeStyle,
|
||||
|
|
|
|||
|
|
@ -1,56 +1,40 @@
|
|||
"use client";
|
||||
|
||||
import Link from "next/link";
|
||||
|
||||
const brandFont = {
|
||||
fontFamily: "var(--font-display), var(--font-sans), sans-serif",
|
||||
} as const;
|
||||
|
||||
export function Logo({ href = "/", className }: { href?: string; className?: string }) {
|
||||
export function Logo() {
|
||||
return (
|
||||
<div className={className}>
|
||||
<div className="flex items-center gap-3.5 group">
|
||||
<Link href={href} aria-label="ktx documentation home" className="flex items-center no-underline">
|
||||
<span className="relative flex items-center justify-center transition-transform duration-300 ease-out group-hover:rotate-[-4deg]">
|
||||
<img
|
||||
src="/ktx/brand/ktx-mascot.svg"
|
||||
alt=""
|
||||
aria-hidden="true"
|
||||
className="h-20 w-20 object-contain block dark:hidden"
|
||||
/>
|
||||
<img
|
||||
src="/ktx/brand/ktx-mascot-dark.svg"
|
||||
alt=""
|
||||
aria-hidden="true"
|
||||
className="h-20 w-20 object-contain hidden dark:block"
|
||||
/>
|
||||
</span>
|
||||
</Link>
|
||||
<div className="flex flex-col items-start leading-none">
|
||||
<Link
|
||||
href={href}
|
||||
className="text-[42px] font-semibold text-fd-foreground tracking-tight no-underline"
|
||||
style={brandFont}
|
||||
>
|
||||
ktx
|
||||
</Link>
|
||||
<a
|
||||
href="https://www.kaelio.com"
|
||||
target="_blank"
|
||||
rel="noreferrer"
|
||||
className="mt-1 whitespace-nowrap text-[13px] font-medium text-fd-muted-foreground/80 tracking-tight no-underline transition-colors hover:text-fd-foreground"
|
||||
style={brandFont}
|
||||
>
|
||||
by Kaelio
|
||||
</a>
|
||||
</div>
|
||||
<div className="flex items-center gap-3.5 group">
|
||||
<div className="relative flex items-center justify-center transition-transform duration-300 ease-out group-hover:rotate-[-4deg]">
|
||||
<img
|
||||
src="/ktx/brand/ktx-mascot.svg"
|
||||
alt=""
|
||||
aria-hidden="true"
|
||||
className="h-20 w-20 object-contain block dark:hidden"
|
||||
/>
|
||||
<img
|
||||
src="/ktx/brand/ktx-mascot-dark.svg"
|
||||
alt=""
|
||||
aria-hidden="true"
|
||||
className="h-20 w-20 object-contain hidden dark:block"
|
||||
/>
|
||||
</div>
|
||||
<div className="flex flex-col items-start leading-none">
|
||||
<span
|
||||
className="text-[19px] font-medium text-fd-muted-foreground/80 tracking-tight border-l border-fd-border pl-3 ml-1"
|
||||
style={brandFont}
|
||||
className="text-[42px] font-semibold text-fd-foreground tracking-tight"
|
||||
style={{ fontFamily: "var(--font-display), var(--font-sans), sans-serif" }}
|
||||
>
|
||||
Docs
|
||||
ktx
|
||||
</span>
|
||||
<span
|
||||
className="mt-1 whitespace-nowrap text-[13px] font-medium text-fd-muted-foreground/80 tracking-tight"
|
||||
style={{ fontFamily: "var(--font-display), var(--font-sans), sans-serif" }}
|
||||
>
|
||||
by Kaelio
|
||||
</span>
|
||||
</div>
|
||||
<span
|
||||
className="text-[19px] font-medium text-fd-muted-foreground/80 tracking-tight border-l border-fd-border pl-3 ml-1"
|
||||
style={{ fontFamily: "var(--font-display), var(--font-sans), sans-serif" }}
|
||||
>
|
||||
Docs
|
||||
</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,576 +0,0 @@
|
|||
"use client";
|
||||
|
||||
import {
|
||||
type Edge,
|
||||
type EdgeProps,
|
||||
getSmoothStepPath,
|
||||
Handle,
|
||||
MarkerType,
|
||||
type Node,
|
||||
type NodeProps,
|
||||
Position,
|
||||
} from "@xyflow/react";
|
||||
|
||||
import { FlowCanvas } from "./flow-canvas";
|
||||
|
||||
type AgentNodeData = {
|
||||
title: string;
|
||||
items: string[];
|
||||
};
|
||||
|
||||
type HubNodeData = {
|
||||
title: string;
|
||||
badge: string;
|
||||
rows: string[];
|
||||
};
|
||||
|
||||
type TargetNodeData = {
|
||||
accent: string;
|
||||
title: string;
|
||||
body: string;
|
||||
rows: { text: string; color?: string; mono?: boolean }[];
|
||||
badge?: string;
|
||||
};
|
||||
|
||||
type AgentNode = Node<AgentNodeData, "agent">;
|
||||
type HubNode = Node<HubNodeData, "hub">;
|
||||
type TargetNode = Node<TargetNodeData, "target">;
|
||||
type FlowNode = AgentNode | HubNode | TargetNode;
|
||||
|
||||
const AGENT_W = 252;
|
||||
const AGENT_H = 96;
|
||||
const HUB_W = 306;
|
||||
const HUB_H = 190;
|
||||
const TARGET_W = 268;
|
||||
const TARGET_H = 148;
|
||||
|
||||
const CENTER_X = 470;
|
||||
const ROW_AGENT_Y = 0;
|
||||
const ROW_HUB_Y = 196;
|
||||
const ROW_TARGET_Y = 488;
|
||||
|
||||
const AGENT_X = CENTER_X - AGENT_W / 2;
|
||||
const HUB_X = CENTER_X - HUB_W / 2;
|
||||
|
||||
const TARGET_GAP_X = 38;
|
||||
const TARGETS_TOTAL = TARGET_W * 2 + TARGET_GAP_X;
|
||||
const TARGETS_START_X = CENTER_X - TARGETS_TOTAL / 2;
|
||||
const CONTEXT_X = TARGETS_START_X;
|
||||
const WAREHOUSE_X = TARGETS_START_X + TARGET_W + TARGET_GAP_X;
|
||||
|
||||
const EDGE_STROKE = "#94a3b8";
|
||||
const CYCLE_STROKE = "#0e7490";
|
||||
const EMERALD = "#059669";
|
||||
const TEAL = "#0e7490";
|
||||
|
||||
const nodes: FlowNode[] = [
|
||||
{
|
||||
id: "agent",
|
||||
type: "agent",
|
||||
position: { x: AGENT_X, y: ROW_AGENT_Y },
|
||||
data: {
|
||||
title: "Your agent",
|
||||
items: ["Claude Code", "Cursor", "Codex"],
|
||||
},
|
||||
draggable: false,
|
||||
selectable: false,
|
||||
},
|
||||
{
|
||||
id: "hub",
|
||||
type: "hub",
|
||||
position: { x: HUB_X, y: ROW_HUB_Y },
|
||||
data: {
|
||||
title: "ktx",
|
||||
badge: "MCP + CLI",
|
||||
rows: [
|
||||
"Search wiki + semantic layer",
|
||||
"Return approved metrics",
|
||||
"Compile metrics → SQL",
|
||||
],
|
||||
},
|
||||
draggable: false,
|
||||
selectable: false,
|
||||
},
|
||||
{
|
||||
id: "context",
|
||||
type: "target",
|
||||
position: { x: CONTEXT_X, y: ROW_TARGET_Y },
|
||||
data: {
|
||||
accent: TEAL,
|
||||
title: "Context layer",
|
||||
body: "Approved definitions agents search before they answer.",
|
||||
rows: [
|
||||
{ text: "wiki/*.md", color: EMERALD, mono: true },
|
||||
{ text: "semantic-layer/*.yaml", color: TEAL, mono: true },
|
||||
],
|
||||
},
|
||||
draggable: false,
|
||||
selectable: false,
|
||||
},
|
||||
{
|
||||
id: "warehouse",
|
||||
type: "target",
|
||||
position: { x: WAREHOUSE_X, y: ROW_TARGET_Y },
|
||||
data: {
|
||||
accent: "#334155",
|
||||
title: "Database",
|
||||
badge: "read-only",
|
||||
body: "Runs the compiled SQL. ktx never writes to it.",
|
||||
rows: [],
|
||||
},
|
||||
draggable: false,
|
||||
selectable: false,
|
||||
},
|
||||
];
|
||||
|
||||
const labelBg = {
|
||||
labelBgPadding: [6, 3] as [number, number],
|
||||
labelBgBorderRadius: 4,
|
||||
labelStyle: {
|
||||
fontSize: 13,
|
||||
fontWeight: 600,
|
||||
fill: "var(--color-fd-muted-foreground)",
|
||||
},
|
||||
labelBgStyle: {
|
||||
fill: "var(--color-fd-background)",
|
||||
stroke: "var(--color-fd-border)",
|
||||
strokeWidth: 1,
|
||||
},
|
||||
};
|
||||
|
||||
const requestMarker = {
|
||||
type: MarkerType.ArrowClosed,
|
||||
color: EDGE_STROKE,
|
||||
width: 16,
|
||||
height: 16,
|
||||
};
|
||||
|
||||
const flowEdges: Edge[] = [
|
||||
{
|
||||
id: "e-ask",
|
||||
source: "agent",
|
||||
sourceHandle: "ask",
|
||||
target: "hub",
|
||||
targetHandle: "ask",
|
||||
type: "straight",
|
||||
label: "ask",
|
||||
...labelBg,
|
||||
style: { stroke: EDGE_STROKE, strokeWidth: 1.5 },
|
||||
markerEnd: requestMarker,
|
||||
},
|
||||
{
|
||||
id: "e-answer",
|
||||
source: "hub",
|
||||
sourceHandle: "answer",
|
||||
target: "agent",
|
||||
targetHandle: "answer",
|
||||
type: "straight",
|
||||
label: "answer",
|
||||
...labelBg,
|
||||
style: { stroke: EDGE_STROKE, strokeWidth: 1.5 },
|
||||
markerEnd: requestMarker,
|
||||
},
|
||||
{
|
||||
id: "e-search",
|
||||
source: "hub",
|
||||
sourceHandle: "to-context",
|
||||
target: "context",
|
||||
targetHandle: "in",
|
||||
type: "smoothstep",
|
||||
label: "search + read",
|
||||
...labelBg,
|
||||
style: { stroke: CYCLE_STROKE, strokeWidth: 1.5 },
|
||||
markerStart: { type: MarkerType.ArrowClosed, color: CYCLE_STROKE, width: 14, height: 14 },
|
||||
markerEnd: { type: MarkerType.ArrowClosed, color: CYCLE_STROKE, width: 14, height: 14 },
|
||||
},
|
||||
{
|
||||
id: "e-readonly",
|
||||
source: "hub",
|
||||
sourceHandle: "to-warehouse",
|
||||
target: "warehouse",
|
||||
targetHandle: "in",
|
||||
type: "smoothstep",
|
||||
label: "read-only",
|
||||
...labelBg,
|
||||
style: { stroke: CYCLE_STROKE, strokeWidth: 1.5 },
|
||||
markerStart: { type: MarkerType.ArrowClosed, color: CYCLE_STROKE, width: 14, height: 14 },
|
||||
markerEnd: { type: MarkerType.ArrowClosed, color: CYCLE_STROKE, width: 14, height: 14 },
|
||||
},
|
||||
];
|
||||
|
||||
function AgentNodeView({ data }: NodeProps<AgentNode>) {
|
||||
return (
|
||||
<div
|
||||
style={{ width: AGENT_W, height: AGENT_H }}
|
||||
className="flex flex-col justify-center rounded-md border border-fd-border bg-fd-card px-3.5 py-2.5 shadow-sm"
|
||||
>
|
||||
<Handle
|
||||
id="ask"
|
||||
type="source"
|
||||
position={Position.Bottom}
|
||||
className="!opacity-0"
|
||||
style={{ left: "35%" }}
|
||||
/>
|
||||
<Handle
|
||||
id="answer"
|
||||
type="target"
|
||||
position={Position.Bottom}
|
||||
className="!opacity-0"
|
||||
style={{ left: "65%" }}
|
||||
/>
|
||||
<div className="flex items-center gap-2.5">
|
||||
<span className="flex h-8 w-8 flex-none items-center justify-center rounded-full bg-fd-primary/15 text-fd-primary">
|
||||
<svg
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
width="18"
|
||||
height="18"
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
strokeWidth="1.75"
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
aria-hidden="true"
|
||||
>
|
||||
<rect x="3" y="6" width="18" height="12" rx="3" />
|
||||
<circle cx="9" cy="12" r="1.25" fill="currentColor" stroke="none" />
|
||||
<circle cx="15" cy="12" r="1.25" fill="currentColor" stroke="none" />
|
||||
<path d="M12 3v3" />
|
||||
</svg>
|
||||
</span>
|
||||
<p className="text-[17px] font-semibold leading-6 text-fd-foreground">
|
||||
{data.title}
|
||||
</p>
|
||||
</div>
|
||||
<div className="mt-2 flex flex-wrap gap-1.5">
|
||||
{data.items.map((item) => (
|
||||
<span
|
||||
key={item}
|
||||
className="rounded border border-fd-border bg-fd-background px-1.5 py-0.5 text-[12px] leading-5 text-fd-muted-foreground"
|
||||
>
|
||||
{item}
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function HubNodeView({ data }: NodeProps<HubNode>) {
|
||||
return (
|
||||
<div
|
||||
style={{ width: HUB_W, height: HUB_H }}
|
||||
className="relative flex flex-col rounded-md border border-cyan-200/20 bg-[#0f1f23] px-4 py-3.5 text-white shadow-sm dark:bg-[#0b181b]"
|
||||
>
|
||||
<Handle
|
||||
id="ask"
|
||||
type="target"
|
||||
position={Position.Top}
|
||||
className="!opacity-0"
|
||||
style={{ left: "37.5%" }}
|
||||
/>
|
||||
<Handle
|
||||
id="answer"
|
||||
type="source"
|
||||
position={Position.Top}
|
||||
className="!opacity-0"
|
||||
style={{ left: "62.5%" }}
|
||||
/>
|
||||
<Handle
|
||||
id="to-context"
|
||||
type="source"
|
||||
position={Position.Bottom}
|
||||
className="!opacity-0"
|
||||
style={{ left: "44%" }}
|
||||
/>
|
||||
<Handle
|
||||
id="to-warehouse"
|
||||
type="source"
|
||||
position={Position.Bottom}
|
||||
className="!opacity-0"
|
||||
style={{ left: "56%" }}
|
||||
/>
|
||||
<div className="flex items-center gap-2.5">
|
||||
<span className="flex h-7 w-7 flex-none items-center justify-center rounded-md bg-cyan-300/95 font-mono text-sm font-bold text-[#0b1c20]">
|
||||
k
|
||||
</span>
|
||||
<span className="text-[19px] font-bold leading-6 text-white">
|
||||
{data.title}
|
||||
</span>
|
||||
<span className="ml-1 rounded border border-cyan-200/30 bg-white/5 px-1.5 py-0.5 font-mono text-[11px] leading-5 text-cyan-100/85">
|
||||
{data.badge}
|
||||
</span>
|
||||
</div>
|
||||
<div className="mt-3 flex flex-1 flex-col justify-center gap-2">
|
||||
{data.rows.map((row) => (
|
||||
<div key={row} className="flex items-center gap-2.5">
|
||||
<span className="h-1.5 w-1.5 flex-none rounded-full bg-cyan-300/95" />
|
||||
<span className="text-[14px] font-medium leading-5 text-cyan-50/90">
|
||||
{row}
|
||||
</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function TargetNodeView({ data }: NodeProps<TargetNode>) {
|
||||
return (
|
||||
<div
|
||||
style={{
|
||||
width: TARGET_W,
|
||||
height: TARGET_H,
|
||||
borderTop: `3px solid ${data.accent}`,
|
||||
}}
|
||||
className="overflow-hidden rounded-md border border-fd-border bg-fd-card px-3.5 py-3 shadow-sm"
|
||||
>
|
||||
<Handle id="in" type="target" position={Position.Top} className="!opacity-0" />
|
||||
<div className="flex items-center gap-2">
|
||||
<p className="text-[17px] font-semibold leading-6 text-fd-foreground">
|
||||
{data.title}
|
||||
</p>
|
||||
{data.badge ? (
|
||||
<span
|
||||
className="rounded-full px-1.5 py-0.5 text-[11px] font-semibold leading-5"
|
||||
style={{
|
||||
color: data.accent,
|
||||
background: "color-mix(in oklch, var(--color-fd-card) 86%, #64748b)",
|
||||
}}
|
||||
>
|
||||
{data.badge}
|
||||
</span>
|
||||
) : null}
|
||||
</div>
|
||||
{data.rows.length > 0 ? (
|
||||
<div className="mt-1 flex flex-col gap-0.5">
|
||||
{data.rows.map((row) => (
|
||||
<span
|
||||
key={row.text}
|
||||
className={
|
||||
row.mono
|
||||
? "font-mono text-[13px] font-semibold tracking-tight"
|
||||
: "text-[12px] leading-4 text-fd-muted-foreground"
|
||||
}
|
||||
style={row.color ? { color: row.color } : undefined}
|
||||
>
|
||||
{row.text}
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
) : null}
|
||||
<p className="mt-1.5 line-clamp-2 text-[13px] leading-[18px] text-fd-muted-foreground">
|
||||
{data.body}
|
||||
</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
/* ------------------------------- Particles ------------------------------- */
|
||||
|
||||
const PARTICLE_SPEED_PX_PER_SEC = 150;
|
||||
const PARTICLE_MIN_DURATION_SEC = 5;
|
||||
|
||||
type Leg = {
|
||||
sx: number;
|
||||
sy: number;
|
||||
sPos: Position;
|
||||
tx: number;
|
||||
ty: number;
|
||||
tPos: Position;
|
||||
};
|
||||
|
||||
const AGENT_ASK_X = AGENT_X + AGENT_W * 0.35;
|
||||
const AGENT_ANSWER_X = AGENT_X + AGENT_W * 0.65;
|
||||
const AGENT_BOTTOM_Y = ROW_AGENT_Y + AGENT_H;
|
||||
const HUB_ASK_X = HUB_X + HUB_W * 0.375;
|
||||
const HUB_ANSWER_X = HUB_X + HUB_W * 0.625;
|
||||
const HUB_TO_CONTEXT_X = HUB_X + HUB_W * 0.44;
|
||||
const HUB_TO_WAREHOUSE_X = HUB_X + HUB_W * 0.56;
|
||||
const HUB_BOTTOM_Y = ROW_HUB_Y + HUB_H;
|
||||
const CONTEXT_TOP_X = CONTEXT_X + TARGET_W / 2;
|
||||
const WAREHOUSE_TOP_X = WAREHOUSE_X + TARGET_W / 2;
|
||||
|
||||
function buildCyclePath(spokeX: number, targetX: number): {
|
||||
d: string;
|
||||
length: number;
|
||||
} {
|
||||
const legs: Leg[] = [
|
||||
// agent → hub (ask, down)
|
||||
{ sx: AGENT_ASK_X, sy: AGENT_BOTTOM_Y, sPos: Position.Bottom, tx: HUB_ASK_X, ty: ROW_HUB_Y, tPos: Position.Top },
|
||||
// through the hub to its spoke handle (down, drawn behind the hub)
|
||||
{ sx: HUB_ASK_X, sy: ROW_HUB_Y, sPos: Position.Bottom, tx: spokeX, ty: HUB_BOTTOM_Y, tPos: Position.Top },
|
||||
// hub → target (down)
|
||||
{ sx: spokeX, sy: HUB_BOTTOM_Y, sPos: Position.Bottom, tx: targetX, ty: ROW_TARGET_Y, tPos: Position.Top },
|
||||
// target → hub (up)
|
||||
{ sx: targetX, sy: ROW_TARGET_Y, sPos: Position.Top, tx: spokeX, ty: HUB_BOTTOM_Y, tPos: Position.Bottom },
|
||||
// through the hub to its answer handle (up, drawn behind the hub)
|
||||
{ sx: spokeX, sy: HUB_BOTTOM_Y, sPos: Position.Top, tx: HUB_ANSWER_X, ty: ROW_HUB_Y, tPos: Position.Bottom },
|
||||
// hub → agent (answer, up)
|
||||
{ sx: HUB_ANSWER_X, sy: ROW_HUB_Y, sPos: Position.Top, tx: AGENT_ANSWER_X, ty: AGENT_BOTTOM_Y, tPos: Position.Bottom },
|
||||
];
|
||||
|
||||
const segments = legs.map((leg) => {
|
||||
const [segment] = getSmoothStepPath({
|
||||
sourceX: leg.sx,
|
||||
sourceY: leg.sy,
|
||||
sourcePosition: leg.sPos,
|
||||
targetX: leg.tx,
|
||||
targetY: leg.ty,
|
||||
targetPosition: leg.tPos,
|
||||
});
|
||||
return segment;
|
||||
});
|
||||
|
||||
let d = segments[0];
|
||||
for (let i = 1; i < segments.length; i += 1) {
|
||||
d += ` ${segments[i].replace(/^M/, "L")}`;
|
||||
}
|
||||
|
||||
const length = legs.reduce(
|
||||
(sum, leg) => sum + Math.abs(leg.tx - leg.sx) + Math.abs(leg.ty - leg.sy),
|
||||
0,
|
||||
);
|
||||
|
||||
return { d, length };
|
||||
}
|
||||
|
||||
type ParticleEdgeData = {
|
||||
d: string;
|
||||
duration: number;
|
||||
beginOffset: number;
|
||||
color: string;
|
||||
};
|
||||
|
||||
type ParticleEdge = Edge<ParticleEdgeData, "particle">;
|
||||
|
||||
function ParticleEdgeView({ id, data }: EdgeProps<ParticleEdge>) {
|
||||
if (!data) return null;
|
||||
const pathId = `runtime-particle-path-${id}`;
|
||||
return (
|
||||
<>
|
||||
<path id={pathId} d={data.d} fill="none" stroke="none" pointerEvents="none" />
|
||||
<g className="runtime-particle" style={{ color: data.color }}>
|
||||
<circle r={7.5} fill="currentColor" opacity={0.16} />
|
||||
<circle r={3.75} fill="currentColor" opacity={0.32} />
|
||||
<circle r={2.1} fill="currentColor" />
|
||||
<animateMotion
|
||||
dur={`${data.duration.toFixed(2)}s`}
|
||||
begin={`-${data.beginOffset.toFixed(2)}s`}
|
||||
repeatCount="indefinite"
|
||||
>
|
||||
<mpath href={`#${pathId}`} />
|
||||
</animateMotion>
|
||||
</g>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
function makeCycleEdge(
|
||||
id: string,
|
||||
source: string,
|
||||
spokeX: number,
|
||||
targetX: number,
|
||||
beginFraction: number,
|
||||
): ParticleEdge {
|
||||
const { d, length } = buildCyclePath(spokeX, targetX);
|
||||
const duration = Math.max(
|
||||
PARTICLE_MIN_DURATION_SEC,
|
||||
length / PARTICLE_SPEED_PX_PER_SEC,
|
||||
);
|
||||
return {
|
||||
id,
|
||||
source,
|
||||
target: source,
|
||||
type: "particle",
|
||||
data: { d, duration, beginOffset: duration * beginFraction, color: CYCLE_STROKE },
|
||||
};
|
||||
}
|
||||
|
||||
const particleEdges: ParticleEdge[] = [
|
||||
makeCycleEdge("p-context", "context", HUB_TO_CONTEXT_X, CONTEXT_TOP_X, 0),
|
||||
makeCycleEdge("p-warehouse", "warehouse", HUB_TO_WAREHOUSE_X, WAREHOUSE_TOP_X, 0.5),
|
||||
];
|
||||
|
||||
const nodeTypes = {
|
||||
agent: AgentNodeView,
|
||||
hub: HubNodeView,
|
||||
target: TargetNodeView,
|
||||
};
|
||||
|
||||
const edgeTypes = {
|
||||
particle: ParticleEdgeView,
|
||||
};
|
||||
|
||||
const edges = [...flowEdges, ...particleEdges];
|
||||
|
||||
export function ProductRuntime() {
|
||||
return (
|
||||
<section
|
||||
className="not-prose my-12 w-full max-w-full min-w-0 space-y-5"
|
||||
aria-labelledby="runtime-title"
|
||||
>
|
||||
<div className="max-w-3xl">
|
||||
<h2
|
||||
id="runtime-title"
|
||||
className="text-xl font-semibold tracking-normal text-fd-foreground sm:text-2xl"
|
||||
style={{ fontFamily: "var(--font-display)" }}
|
||||
>
|
||||
How serving works
|
||||
</h2>
|
||||
<p className="mt-3 text-sm leading-6 text-fd-muted-foreground">
|
||||
At runtime, agents reach ktx through MCP. ktx searches the context
|
||||
layer, returns approved metrics, and compiles them into read-only SQL
|
||||
the warehouse runs.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<article
|
||||
className="max-w-full min-w-0 overflow-hidden rounded-lg border border-fd-border bg-fd-card shadow-sm"
|
||||
aria-label="ktx serving flow from an agent request to a governed answer"
|
||||
>
|
||||
<div className="border-b border-fd-border bg-fd-muted/35 px-5 py-4">
|
||||
<p className="text-xs font-semibold uppercase tracking-wide text-fd-primary">
|
||||
Serving flow
|
||||
</p>
|
||||
<h3
|
||||
className="mt-1 text-base font-semibold tracking-normal text-fd-foreground sm:text-lg"
|
||||
style={{ fontFamily: "var(--font-display)" }}
|
||||
>
|
||||
From an agent request to a governed answer
|
||||
</h3>
|
||||
<p className="mt-2 max-w-3xl text-xs leading-5 text-fd-muted-foreground">
|
||||
The agent asks in plain language. ktx is the only thing that touches
|
||||
the context layer and the warehouse, and every database connection
|
||||
is read-only.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<FlowCanvas
|
||||
nodes={nodes}
|
||||
edges={edges}
|
||||
nodeTypes={nodeTypes}
|
||||
edgeTypes={edgeTypes}
|
||||
canvasStyle={{
|
||||
height: "min(620px, 98vw)",
|
||||
minHeight: 430,
|
||||
}}
|
||||
className="runtime-canvas"
|
||||
fitViewOptions={{ padding: 0.06 }}
|
||||
ariaLabel="ktx serving flow diagram"
|
||||
/>
|
||||
</article>
|
||||
<style>{`
|
||||
.runtime-canvas .runtime-particle {
|
||||
pointer-events: none;
|
||||
filter: drop-shadow(0 0 6px currentColor);
|
||||
}
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.runtime-canvas .runtime-particle {
|
||||
display: none;
|
||||
}
|
||||
}
|
||||
`}</style>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
|
@ -177,9 +177,7 @@ Slowest phase: reconciliation (2m 05s, 48% of wall time). 2 work units (1 failed
|
|||
|
||||
Work units run serially by default (`ingest.workUnits.maxConcurrency` is `1`);
|
||||
raise it in `ktx.yaml` if the profile shows the run is bound by serialized
|
||||
work-unit agent loops. If the provider reports an LLM rate limit, **ktx** shows
|
||||
a transient wait message and temporarily reduces effective work-unit concurrency
|
||||
according to `ingest.rateLimit`.
|
||||
work-unit agent loops.
|
||||
|
||||
## Common errors
|
||||
|
||||
|
|
|
|||
|
|
@ -74,44 +74,6 @@ The public context-build entrypoint is `ktx ingest [connectionId]` or
|
|||
| `-v`, `--version` | Show the CLI package name and version. |
|
||||
| `-h`, `--help` | Show help for the current command. |
|
||||
|
||||
## Update notices
|
||||
|
||||
> **Note:** The update notifier writes only to stderr and keeps command stdout
|
||||
> unchanged.
|
||||
|
||||
When a newer package is available on your installed release channel, `ktx`
|
||||
prints a short notice after the command finishes:
|
||||
|
||||
```text
|
||||
↑ Update available: ktx 0.9.0 → 0.10.0
|
||||
npm i -g @kaelio/ktx
|
||||
```
|
||||
|
||||
Stable installs compare against the npm `latest` dist-tag.
|
||||
Release-candidate installs compare against the `next` dist-tag and show:
|
||||
|
||||
```text
|
||||
npm i -g @kaelio/ktx@next
|
||||
```
|
||||
|
||||
The check is skipped for JSON output, CI, non-TTY stdout, and hidden completion
|
||||
commands. To opt out explicitly, set any of these environment variables:
|
||||
|
||||
```bash
|
||||
KTX_NO_UPDATE_CHECK=1
|
||||
NO_UPDATE_NOTIFIER=1
|
||||
DO_NOT_TRACK=1
|
||||
```
|
||||
|
||||
The `ktx` CLI prints one npm command because globally installed binaries don't
|
||||
expose a reliable runtime package-manager signal. If you prefer another global
|
||||
package manager, use the equivalent command:
|
||||
|
||||
```bash
|
||||
pnpm add -g @kaelio/ktx
|
||||
yarn global add @kaelio/ktx
|
||||
```
|
||||
|
||||
## Project resolution
|
||||
|
||||
Most commands are project-aware. Pass `--project-dir <path>` when scripting or
|
||||
|
|
|
|||
|
|
@ -46,33 +46,6 @@ an operation errors, the detail we record is the error as your tools reported
|
|||
it, which can include identifiers from your setup. If you'd rather send nothing
|
||||
at all, turn telemetry off using any of the options above.
|
||||
|
||||
## Error reports
|
||||
|
||||
When telemetry is enabled, **ktx** sends PostHog Error Tracking `$exception`
|
||||
events for CLI and daemon exceptions. Error reports help group crashes and
|
||||
handled failures into PostHog issues.
|
||||
|
||||
Error reports can include:
|
||||
|
||||
- Stack frames, including function names, local file paths, line numbers, and
|
||||
SDK-provided source context.
|
||||
- Error class names and raw error messages.
|
||||
- Cause chains when the runtime exposes them.
|
||||
- `source`, `handled`, and `fatal` diagnostic fields.
|
||||
- Runtime version, OS, architecture, and CI fields.
|
||||
- The hashed `projectId` when **ktx** knows the project.
|
||||
|
||||
Error reports never intentionally include:
|
||||
|
||||
- Secrets, credentials, API keys, tokens, cookies, signed URLs, or auth headers.
|
||||
- Database URLs, connection strings, DSNs, raw argv, or raw environment values.
|
||||
- SQL text, schema names, table names, or column names as explicit payload
|
||||
properties.
|
||||
- Customer row data.
|
||||
- User prompt text or raw MCP arguments.
|
||||
|
||||
The same opt-out controls listed above disable error reports.
|
||||
|
||||
## Storage and retention
|
||||
|
||||
Telemetry is sent to PostHog, a third-party product-analytics service used by
|
||||
|
|
|
|||
|
|
@ -452,16 +452,6 @@ ingest:
|
|||
stepBudget: 40
|
||||
maxConcurrency: 2
|
||||
failureMode: continue
|
||||
rateLimit:
|
||||
enabled: true
|
||||
throttleThreshold: 0.8
|
||||
minConcurrencyUnderPressure: 1
|
||||
maxWaitMs: 600000
|
||||
retry:
|
||||
maxAttempts: 6
|
||||
baseDelayMs: 1000
|
||||
maxDelayMs: 60000
|
||||
jitter: true
|
||||
```
|
||||
|
||||
### Adapters
|
||||
|
|
@ -508,24 +498,6 @@ handles failures.
|
|||
| `workUnits.maxConcurrency` | `int > 0` | `1` | How many work units run in parallel. |
|
||||
| `workUnits.failureMode` | `abort` \| `continue` | `continue` | `abort` stops the whole ingest run on the first failure; `continue` records it and keeps going. |
|
||||
|
||||
### Rate limits
|
||||
|
||||
`rateLimit` controls provider-neutral pacing for LLM calls during ingest. When a
|
||||
provider reports a subscription window, retry-after delay, or HTTP 429,
|
||||
**ktx** pauses new work-unit model calls, shows a transient wait in the CLI,
|
||||
and reduces work-unit concurrency while the provider is under pressure.
|
||||
|
||||
| Field | Type | Default | Purpose |
|
||||
|-------|------|---------|---------|
|
||||
| `rateLimit.enabled` | `boolean` | `true` | Master switch for ingest LLM rate-limit pacing and visible waits. |
|
||||
| `rateLimit.throttleThreshold` | `number between 0 and 1` | `0.8` | Fraction of a known provider window at which **ktx** starts reducing concurrency. |
|
||||
| `rateLimit.minConcurrencyUnderPressure` | `int > 0` | `1` | Effective work-unit concurrency while a provider is under rate-limit pressure. |
|
||||
| `rateLimit.maxWaitMs` | `int > 0` | unset | Caps how long a single provider-reset wait can last. This bounds each wait, not the whole run: after a capped wait elapses **ktx** retries and may pause again. Omit to wait until the provider's reset time. |
|
||||
| `rateLimit.retry.maxAttempts` | `int > 0` | `6` | Maximum attempts for a single rate-limited LLM call before the failure surfaces (counts the first try). Also bounds how far opaque backoff grows for responses without a reset time or retry-after value. |
|
||||
| `rateLimit.retry.baseDelayMs` | `int > 0` | `1000` | Initial opaque retry delay in milliseconds. |
|
||||
| `rateLimit.retry.maxDelayMs` | `int > 0` | `60000` | Maximum opaque retry delay in milliseconds. |
|
||||
| `rateLimit.retry.jitter` | `boolean` | `true` | Add jitter to opaque retry delays. |
|
||||
|
||||
## `scan`
|
||||
|
||||
`scan` configures how schema-level inputs become structured context:
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ description: ktx is an open-source, self-improving context layer for data agents
|
|||
---
|
||||
|
||||
import { ProductMechanics } from "@/components/product-mechanics";
|
||||
import { ProductRuntime } from "@/components/product-runtime";
|
||||
|
||||
<div className="not-prose mb-10">
|
||||
<div>
|
||||
|
|
@ -60,8 +59,6 @@ serves that context to agents at runtime.
|
|||
|
||||
<ProductMechanics />
|
||||
|
||||
<ProductRuntime />
|
||||
|
||||
## Use it for
|
||||
|
||||
Use **ktx** when agents need more than raw database access. Agents can search wiki
|
||||
|
|
|
|||
|
|
@ -30,36 +30,7 @@ const config = {
|
|||
};
|
||||
},
|
||||
async redirects() {
|
||||
// Alias-host canonicalization MUST come before the generic root/docs
|
||||
// redirects below. Those generic rules have no host guard, so if they ran
|
||||
// first they would inject a "/ktx" basePath into the path on the alias
|
||||
// hosts, which the alias catch-alls would then prepend a second time —
|
||||
// producing https://docs.kaelio.com/ktx/ktx/docs/... Redirects also run
|
||||
// before beforeFiles rewrites, so the ktx.sh catch-all must exclude
|
||||
// /stars* to let the stars dashboard rewrite proxy through.
|
||||
return [
|
||||
{
|
||||
source: "/slack",
|
||||
has: [{ type: "host", value: "ktx.sh" }],
|
||||
destination:
|
||||
"https://join.slack.com/t/ktxcommunity/shared_invite/zt-3y9b44m1x-LVyNNJD5nwaZHq4XS29LMQ",
|
||||
permanent: false,
|
||||
basePath: false,
|
||||
},
|
||||
{
|
||||
source: "/:path*",
|
||||
has: [{ type: "host", value: "docs.ktx.sh" }],
|
||||
destination: "https://docs.kaelio.com/ktx/:path*",
|
||||
permanent: true,
|
||||
basePath: false,
|
||||
},
|
||||
{
|
||||
source: "/:path((?!stars(?:/|$)).*)",
|
||||
has: [{ type: "host", value: "ktx.sh" }],
|
||||
destination: "https://docs.kaelio.com/ktx/:path",
|
||||
permanent: true,
|
||||
basePath: false,
|
||||
},
|
||||
{
|
||||
source: "/",
|
||||
destination: "/ktx/docs/getting-started/introduction",
|
||||
|
|
@ -72,6 +43,28 @@ const config = {
|
|||
permanent: false,
|
||||
basePath: false,
|
||||
},
|
||||
{
|
||||
source: "/:path*",
|
||||
has: [{ type: "host", value: "docs.ktx.sh" }],
|
||||
destination: "https://docs.kaelio.com/ktx/:path*",
|
||||
permanent: true,
|
||||
basePath: false,
|
||||
},
|
||||
{
|
||||
source: "/slack",
|
||||
has: [{ type: "host", value: "ktx.sh" }],
|
||||
destination:
|
||||
"https://join.slack.com/t/ktxcommunity/shared_invite/zt-3y9b44m1x-LVyNNJD5nwaZHq4XS29LMQ",
|
||||
permanent: false,
|
||||
basePath: false,
|
||||
},
|
||||
{
|
||||
source: "/:path((?!stars(?:/|$)).*)",
|
||||
has: [{ type: "host", value: "ktx.sh" }],
|
||||
destination: "https://docs.kaelio.com/ktx/:path",
|
||||
permanent: true,
|
||||
basePath: false,
|
||||
},
|
||||
];
|
||||
},
|
||||
};
|
||||
|
|
|
|||
|
|
@ -2,8 +2,6 @@ import assert from "node:assert/strict";
|
|||
import { spawn } from "node:child_process";
|
||||
import { once } from "node:events";
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import http from "node:http";
|
||||
import https from "node:https";
|
||||
import { dirname, join } from "node:path";
|
||||
import { createServer } from "node:net";
|
||||
import { after, before, test } from "node:test";
|
||||
|
|
@ -102,37 +100,6 @@ after(async () => {
|
|||
}
|
||||
});
|
||||
|
||||
// Node's fetch (undici) overwrites the Host header with the connection host,
|
||||
// so the alias-host redirect rules never match. The low-level http(s) client
|
||||
// sends Host verbatim, which is what the alias canonicalization keys off of.
|
||||
function requestWithHost(hostHeader, path) {
|
||||
const target = new URL(docsSiteUrl);
|
||||
const client = target.protocol === "https:" ? https : http;
|
||||
const port =
|
||||
target.port || (target.protocol === "https:" ? "443" : "80");
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const request = client.request(
|
||||
{
|
||||
hostname: target.hostname,
|
||||
port,
|
||||
path,
|
||||
method: "GET",
|
||||
headers: { Host: hostHeader },
|
||||
},
|
||||
(response) => {
|
||||
response.resume();
|
||||
resolve({
|
||||
status: response.statusCode,
|
||||
location: response.headers.location,
|
||||
});
|
||||
},
|
||||
);
|
||||
request.on("error", reject);
|
||||
request.end();
|
||||
});
|
||||
}
|
||||
|
||||
test("/ktx/docs redirects to the docs introduction", async () => {
|
||||
const response = await fetch(`${docsSiteUrl}${docsBasePath}/docs`, {
|
||||
redirect: "manual",
|
||||
|
|
@ -174,51 +141,3 @@ test("/ktx/api/search returns docs search results", async () => {
|
|||
"search should return at least one docs result",
|
||||
);
|
||||
});
|
||||
|
||||
test("ktx.sh canonicalizes to a single /ktx basePath on the docs host", async () => {
|
||||
const root = await requestWithHost("ktx.sh", "/");
|
||||
assert.equal(root.status, 308);
|
||||
assert.equal(root.location, "https://docs.kaelio.com/ktx/");
|
||||
assert.ok(
|
||||
!root.location.includes("/ktx/ktx"),
|
||||
"the basePath must not be doubled",
|
||||
);
|
||||
|
||||
const page = await requestWithHost(
|
||||
"ktx.sh",
|
||||
"/docs/getting-started/quickstart",
|
||||
);
|
||||
assert.equal(page.status, 308);
|
||||
assert.equal(
|
||||
page.location,
|
||||
"https://docs.kaelio.com/ktx/docs/getting-started/quickstart",
|
||||
);
|
||||
});
|
||||
|
||||
test("docs.ktx.sh canonicalizes to a single /ktx basePath on the docs host", async () => {
|
||||
const root = await requestWithHost("docs.ktx.sh", "/");
|
||||
assert.equal(root.status, 308);
|
||||
assert.equal(root.location, "https://docs.kaelio.com/ktx");
|
||||
assert.ok(
|
||||
!root.location.includes("/ktx/ktx"),
|
||||
"the basePath must not be doubled",
|
||||
);
|
||||
|
||||
const page = await requestWithHost("docs.ktx.sh", "/llms.txt");
|
||||
assert.equal(page.status, 308);
|
||||
assert.equal(page.location, "https://docs.kaelio.com/ktx/llms.txt");
|
||||
});
|
||||
|
||||
test("ktx.sh keeps the /slack and /stars exceptions", async () => {
|
||||
const slack = await requestWithHost("ktx.sh", "/slack");
|
||||
assert.equal(slack.status, 307);
|
||||
assert.match(slack.location, /^https:\/\/join\.slack\.com\//);
|
||||
|
||||
// /stars is proxied by a beforeFiles rewrite, so the apex catch-all must not
|
||||
// canonicalize it to the docs host.
|
||||
const stars = await requestWithHost("ktx.sh", "/stars");
|
||||
assert.ok(
|
||||
!(stars.location ?? "").startsWith("https://docs.kaelio.com"),
|
||||
"the stars dashboard must not be redirected to the docs host",
|
||||
);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -85,7 +85,7 @@ test("product mechanics component explains ingestion outputs", async () => {
|
|||
"compile into SQL",
|
||||
'"use client"',
|
||||
"@xyflow/react",
|
||||
"<FlowCanvas",
|
||||
"<ReactFlow",
|
||||
"getSmoothStepPath",
|
||||
"animateMotion",
|
||||
"mechanics-particle",
|
||||
|
|
@ -97,21 +97,21 @@ test("product mechanics component explains ingestion outputs", async () => {
|
|||
);
|
||||
}
|
||||
|
||||
// The ReactFlow canvas config lives in the shared FlowCanvas wrapper, which
|
||||
// product-mechanics renders. Assert the static read-only behavior there.
|
||||
const flowCanvas = await readDocsFile("components/flow-canvas.tsx");
|
||||
for (const guard of [
|
||||
assert.match(
|
||||
component,
|
||||
/nodesDraggable=\{false\}/,
|
||||
/nodesConnectable=\{false\}/,
|
||||
"ReactFlow canvas should disable node dragging",
|
||||
);
|
||||
assert.match(
|
||||
component,
|
||||
/panOnDrag=\{false\}/,
|
||||
"ReactFlow canvas should disable panning",
|
||||
);
|
||||
assert.match(
|
||||
component,
|
||||
/zoomOnScroll=\{false\}/,
|
||||
/elementsSelectable=\{false\}/,
|
||||
]) {
|
||||
assert.match(
|
||||
flowCanvas,
|
||||
guard,
|
||||
`shared FlowCanvas should enforce static read-only behavior: ${guard}`,
|
||||
);
|
||||
}
|
||||
"ReactFlow canvas should disable scroll zoom",
|
||||
);
|
||||
|
||||
assert.doesNotMatch(component, /raw-sources/);
|
||||
assert.doesNotMatch(component, /\.ktx/);
|
||||
|
|
|
|||
|
|
@ -1,74 +0,0 @@
|
|||
import assert from "node:assert/strict";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { dirname, join } from "node:path";
|
||||
import { test } from "node:test";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const docsSiteDir = join(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
|
||||
async function readDocsFile(path) {
|
||||
return readFile(join(docsSiteDir, path), "utf8");
|
||||
}
|
||||
|
||||
test("docs introduction renders the serving phase after ingestion", async () => {
|
||||
const introduction = await readDocsFile(
|
||||
"content/docs/getting-started/introduction.mdx",
|
||||
);
|
||||
|
||||
assert.match(
|
||||
introduction,
|
||||
/import\s+\{\s*ProductRuntime\s*\}\s+from\s+"@\/components\/product-runtime";/,
|
||||
);
|
||||
assert.match(introduction, /<ProductRuntime\s*\/>/);
|
||||
|
||||
const mechanicsIndex = introduction.indexOf("<ProductMechanics />");
|
||||
const runtimeIndex = introduction.indexOf("<ProductRuntime />");
|
||||
const useCaseIndex = introduction.indexOf("## Use it for");
|
||||
|
||||
assert.ok(
|
||||
runtimeIndex > mechanicsIndex,
|
||||
"serving diagram should appear after the ingestion diagram",
|
||||
);
|
||||
assert.ok(
|
||||
runtimeIndex < useCaseIndex,
|
||||
"serving diagram should appear before use-case sections",
|
||||
);
|
||||
});
|
||||
|
||||
test("product runtime component explains the serving cycle", async () => {
|
||||
const component = await readDocsFile("components/product-runtime.tsx");
|
||||
|
||||
for (const expectedText of [
|
||||
"How serving works",
|
||||
"Serving flow",
|
||||
"From an agent request to a governed answer",
|
||||
"Your agent",
|
||||
"Claude Code",
|
||||
"Cursor",
|
||||
"Codex",
|
||||
"Search wiki + semantic layer",
|
||||
"Return approved metrics",
|
||||
"Compile metrics → SQL",
|
||||
"Context layer",
|
||||
"Database",
|
||||
"search + read",
|
||||
"read-only",
|
||||
"wiki/*.md",
|
||||
"semantic-layer/*.yaml",
|
||||
'"use client"',
|
||||
"@xyflow/react",
|
||||
"FlowCanvas",
|
||||
"getSmoothStepPath",
|
||||
"animateMotion",
|
||||
"runtime-particle",
|
||||
"buildCyclePath",
|
||||
]) {
|
||||
assert.ok(
|
||||
component.includes(expectedText),
|
||||
`component should include: ${expectedText}`,
|
||||
);
|
||||
}
|
||||
|
||||
assert.doesNotMatch(component, /raw-sources/);
|
||||
assert.doesNotMatch(component, /<img/);
|
||||
});
|
||||
|
|
@ -2,10 +2,6 @@
|
|||
"name": "@kaelio/ktx",
|
||||
"version": "0.9.0",
|
||||
"description": "Standalone ktx context layer for data agents",
|
||||
"author": {
|
||||
"name": "Kaelio",
|
||||
"url": "https://www.kaelio.com"
|
||||
},
|
||||
"type": "module",
|
||||
"engines": {
|
||||
"node": ">=22.0.0"
|
||||
|
|
@ -51,7 +47,6 @@
|
|||
"@ai-sdk/devtools": "0.0.18",
|
||||
"@ai-sdk/google-vertex": "^4.0.134",
|
||||
"@anthropic-ai/claude-agent-sdk": "0.3.146",
|
||||
"@clack/core": "1.3.1",
|
||||
"@clack/prompts": "1.4.0",
|
||||
"@clickhouse/client": "^1.18.5",
|
||||
"@commander-js/extra-typings": "14.0.0",
|
||||
|
|
@ -77,7 +72,6 @@
|
|||
"pg": "^8.21.0",
|
||||
"posthog-node": "^5.34.9",
|
||||
"react": "^19.2.6",
|
||||
"semver": "^7.8.1",
|
||||
"simple-git": "3.36.0",
|
||||
"snowflake-sdk": "^2.4.2",
|
||||
"yaml": "^2.9.0",
|
||||
|
|
@ -91,7 +85,6 @@
|
|||
"@types/node": "^25.9.1",
|
||||
"@types/pg": "^8.20.0",
|
||||
"@types/react": "^19.2.15",
|
||||
"@types/semver": "^7.7.1",
|
||||
"@vitest/coverage-v8": "^4.1.7",
|
||||
"ajv": "8.20.0",
|
||||
"ink-testing-library": "^4.0.0",
|
||||
|
|
|
|||
|
|
@ -3,30 +3,6 @@ import type { KtxCliIo } from './cli-runtime.js';
|
|||
|
||||
const ESC = String.fromCharCode(0x1b);
|
||||
|
||||
export interface CliStyleEnv {
|
||||
NO_COLOR?: string;
|
||||
TERM?: string;
|
||||
}
|
||||
|
||||
function ansiEnabled(env: CliStyleEnv = process.env): boolean {
|
||||
return !env.NO_COLOR && env.TERM !== 'dumb';
|
||||
}
|
||||
|
||||
function ansiColor(text: string, open: number, close: number, env?: CliStyleEnv): string {
|
||||
if (!ansiEnabled(env)) {
|
||||
return text;
|
||||
}
|
||||
return `${ESC}[${open}m${text}${ESC}[${close}m`;
|
||||
}
|
||||
|
||||
export function dim(text: string, env?: CliStyleEnv): string {
|
||||
return ansiColor(text, 2, 22, env);
|
||||
}
|
||||
|
||||
export function cyan(text: string, env?: CliStyleEnv): string {
|
||||
return ansiColor(text, 36, 39, env);
|
||||
}
|
||||
|
||||
export interface RailBufferedSource {
|
||||
stdoutText(): string;
|
||||
stderrText(): string;
|
||||
|
|
@ -85,11 +61,11 @@ export function createClackSpinner(): KtxCliSpinner {
|
|||
}
|
||||
|
||||
function magenta(text: string): string {
|
||||
return ansiColor(text, 35, 39);
|
||||
return `${ESC}[35m${text}${ESC}[39m`;
|
||||
}
|
||||
|
||||
function red(text: string): string {
|
||||
return ansiColor(text, 31, 39);
|
||||
return `${ESC}[31m${text}${ESC}[39m`;
|
||||
}
|
||||
|
||||
export function createStaticCliSpinner(io: KtxCliSpinnerIo): KtxCliSpinner {
|
||||
|
|
|
|||
|
|
@ -16,7 +16,6 @@ import { renderMissingProjectMessage } from './doctor.js';
|
|||
import { findNearestKtxProjectDir, resolveKtxProjectDir } from './project-resolver.js';
|
||||
import { profileMark, profileSpan } from './startup-profile.js';
|
||||
import type { CommandOutcome } from './telemetry/index.js';
|
||||
import { prepareUpdateCheckNotice, type PrepareUpdateCheckNoticeOptions } from './update-check/update-check.js';
|
||||
|
||||
profileMark('module:cli-program');
|
||||
|
||||
|
|
@ -40,8 +39,6 @@ interface KtxCommanderProgramOptions {
|
|||
runInit: (args: { projectDir: string; force: boolean }, io: KtxCliIo) => Promise<number>;
|
||||
}
|
||||
|
||||
type KtxCliUpdateCheckOptions = Pick<PrepareUpdateCheckNoticeOptions, 'env' | 'fetchDistTags' | 'homeDir' | 'now'>;
|
||||
|
||||
export interface BuildKtxProgramOptions {
|
||||
io: KtxCliIo;
|
||||
deps: KtxCliDeps;
|
||||
|
|
@ -50,7 +47,6 @@ export interface BuildKtxProgramOptions {
|
|||
setExitCode?: (code: number) => void;
|
||||
argv?: string[];
|
||||
setTelemetryModule?: (telemetry: typeof import('./telemetry/index.js')) => void;
|
||||
updateCheck?: KtxCliUpdateCheckOptions;
|
||||
}
|
||||
|
||||
type CommanderExitLike = { exitCode: number; code: string; message: string };
|
||||
|
|
@ -435,29 +431,16 @@ export function collectCommandFlagsPresent(command: CommandUnknownOpts): Record<
|
|||
|
||||
export function buildKtxProgram(options: BuildKtxProgramOptions): Command {
|
||||
const program = createBaseProgram(options.packageInfo, options.io);
|
||||
let pendingUpdateNotice: string | null = null;
|
||||
|
||||
program.hook('preAction', async (_thisCommand, actionCommand) => {
|
||||
// The hidden completion command must stay silent and side-effect free: skip
|
||||
// the telemetry notice, command span, project checks, and update checks entirely.
|
||||
// the telemetry notice, command span, and project checks entirely.
|
||||
if (commandPath(actionCommand as CommandPathNode).includes('__complete')) {
|
||||
return;
|
||||
}
|
||||
const commandNode = actionCommand as CommandPathNode;
|
||||
const updateCheck = await prepareUpdateCheckNotice({
|
||||
io: options.io,
|
||||
env: options.updateCheck?.env,
|
||||
fetchDistTags: options.updateCheck?.fetchDistTags,
|
||||
homeDir: options.updateCheck?.homeDir,
|
||||
installedVersion: options.packageInfo.version,
|
||||
now: options.updateCheck?.now,
|
||||
commandOptions: commandOptions(commandNode),
|
||||
});
|
||||
pendingUpdateNotice = updateCheck.notice;
|
||||
|
||||
const telemetry = await import('./telemetry/index.js');
|
||||
options.setTelemetryModule?.(telemetry);
|
||||
await telemetry.showTelemetryNoticeIfNeeded(options.io, options.packageInfo);
|
||||
const commandNode = actionCommand as CommandPathNode;
|
||||
const path = commandPath(commandNode);
|
||||
const projectDir = resolveCommandProjectDir(commandNode);
|
||||
const hasProject = ktxYamlExists(projectDir);
|
||||
|
|
@ -474,13 +457,6 @@ export function buildKtxProgram(options: BuildKtxProgramOptions): Command {
|
|||
ensureProjectAvailable(options.io, commandNode);
|
||||
});
|
||||
|
||||
program.hook('postAction', () => {
|
||||
if (pendingUpdateNotice) {
|
||||
options.io.stderr.write(pendingUpdateNotice);
|
||||
pendingUpdateNotice = null;
|
||||
}
|
||||
});
|
||||
|
||||
const context: KtxCliCommandContext = {
|
||||
io: options.io,
|
||||
deps: options.deps,
|
||||
|
|
@ -553,13 +529,6 @@ export async function runCommanderKtxCli(
|
|||
try {
|
||||
return await runBareInteractiveCommand(program, io, context);
|
||||
} catch (error) {
|
||||
const telemetry = await import('./telemetry/index.js');
|
||||
await telemetry.reportException({
|
||||
error,
|
||||
context: { source: 'bare-interactive', handled: true, fatal: false },
|
||||
packageInfo: info,
|
||||
io,
|
||||
});
|
||||
io.stderr.write(`${formatCliError(error)}\n`);
|
||||
return 1;
|
||||
}
|
||||
|
|
@ -594,23 +563,6 @@ export async function runCommanderKtxCli(
|
|||
outcome: commandOutcomeForParseResult(parseError, exitCode),
|
||||
error: parseError,
|
||||
});
|
||||
if (
|
||||
parseError &&
|
||||
!isCommanderExit(parseError) &&
|
||||
!isKtxProjectMissingAbortError(parseError)
|
||||
) {
|
||||
await telemetryModule.reportException({
|
||||
error: parseError,
|
||||
context: {
|
||||
source: completed?.commandPath.join(' ') ?? 'commander parseAsync',
|
||||
handled: true,
|
||||
fatal: false,
|
||||
},
|
||||
projectDir: completed?.projectGroupAttached ? completed.projectDir : undefined,
|
||||
packageInfo: info,
|
||||
io,
|
||||
});
|
||||
}
|
||||
await telemetryModule.emitCompletedCommand({ completed, packageInfo: info, io });
|
||||
await telemetryModule.shutdownTelemetryEmitter();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -129,48 +129,6 @@ function installTelemetrySignalFlush(io: KtxCliIo, info: KtxCliPackageInfo): ()
|
|||
};
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function createGlobalExceptionReporter(io: KtxCliIo, info: KtxCliPackageInfo) {
|
||||
return async (source: 'uncaughtException' | 'unhandledRejection', error: unknown): Promise<void> => {
|
||||
const { reportException, shutdownTelemetryEmitter } = await import('./telemetry/index.js');
|
||||
await reportException({
|
||||
error,
|
||||
context: { source, handled: false, fatal: true },
|
||||
io,
|
||||
packageInfo: info,
|
||||
immediate: true,
|
||||
});
|
||||
await shutdownTelemetryEmitter();
|
||||
};
|
||||
}
|
||||
|
||||
export function installGlobalExceptionHandlers(io: KtxCliIo, info: KtxCliPackageInfo): () => void {
|
||||
const report = createGlobalExceptionReporter(io, info);
|
||||
const handle = (source: 'uncaughtException' | 'unhandledRejection', error: unknown): void => {
|
||||
void (async () => {
|
||||
try {
|
||||
await report(source, error);
|
||||
} catch {
|
||||
// Best-effort: preserve Node's process termination behavior.
|
||||
}
|
||||
if (error instanceof Error && error.stack) {
|
||||
io.stderr.write(`${error.stack}\n`);
|
||||
} else {
|
||||
io.stderr.write(`${String(error)}\n`);
|
||||
}
|
||||
process.exit(1);
|
||||
})();
|
||||
};
|
||||
const onUncaught = (error: Error): void => handle('uncaughtException', error);
|
||||
const onUnhandled = (reason: unknown): void => handle('unhandledRejection', reason);
|
||||
process.on('uncaughtException', onUncaught);
|
||||
process.on('unhandledRejection', onUnhandled);
|
||||
return () => {
|
||||
process.off('uncaughtException', onUncaught);
|
||||
process.off('unhandledRejection', onUnhandled);
|
||||
};
|
||||
}
|
||||
|
||||
export async function runKtxCli(
|
||||
argv = process.argv.slice(2),
|
||||
io: KtxCliIo = process,
|
||||
|
|
@ -183,14 +141,11 @@ export async function runKtxCli(
|
|||
// Real-process entry only: flush telemetry if interrupted. Test/programmatic
|
||||
// callers pass their own `io`, so they never install process-level handlers.
|
||||
const removeSignalFlush = (io as unknown) === process ? installTelemetrySignalFlush(io, info) : undefined;
|
||||
const removeGlobalExceptionHandlers =
|
||||
(io as unknown) === process ? installGlobalExceptionHandlers(io, info) : undefined;
|
||||
try {
|
||||
return await runCommanderKtxCli(argv, io, deps, info, {
|
||||
runInit: runInitForCommander,
|
||||
});
|
||||
} finally {
|
||||
removeGlobalExceptionHandlers?.();
|
||||
removeSignalFlush?.();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -406,8 +406,6 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo
|
|||
}
|
||||
|
||||
const resolvedAgentScope = options.local ? 'local' : options.global ? 'global' : 'project';
|
||||
const debugEnabled =
|
||||
((command.optsWithGlobals ? command.optsWithGlobals() : command.opts()) as { debug?: unknown }).debug === true;
|
||||
await runSetupArgs(context, {
|
||||
command: 'run',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
|
|
@ -417,7 +415,6 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo
|
|||
agentScope: resolvedAgentScope,
|
||||
skipAgents: options.skipAgents === true,
|
||||
inputMode: options.input === false ? 'disabled' : 'auto',
|
||||
...(debugEnabled ? { debug: true } : {}),
|
||||
yes: options.yes === true,
|
||||
cliVersion: context.packageInfo.version,
|
||||
...(options.llmBackend ? { llmBackend: options.llmBackend } : {}),
|
||||
|
|
|
|||
|
|
@ -16,8 +16,7 @@ import { bold, dim, green, red, SYMBOLS } from './io/symbols.js';
|
|||
import { createKtxCliScanConnector } from './local-scan-connectors.js';
|
||||
import { profileMark } from './startup-profile.js';
|
||||
import { isDemoConnection } from './telemetry/demo-detect.js';
|
||||
import { emitTelemetryEvent, reportException } from './telemetry/index.js';
|
||||
import { collectTelemetryRedactionSecrets } from './telemetry/redaction-secrets.js';
|
||||
import { emitTelemetryEvent } from './telemetry/index.js';
|
||||
import { formatErrorDetail, scrubErrorClass } from './telemetry/scrubber.js';
|
||||
|
||||
profileMark('module:connection');
|
||||
|
|
@ -75,12 +74,6 @@ async function testNativeConnection(
|
|||
}
|
||||
const result = await connector.testConnection();
|
||||
if (!result.success) {
|
||||
// Re-throw the driver's original error so connection_test telemetry records
|
||||
// its real class (e.g. ConnectionError) and code (e.g. ELOGIN) instead of
|
||||
// collapsing every native failure to a generic Error with no code.
|
||||
if (result.cause instanceof Error) {
|
||||
throw result.cause;
|
||||
}
|
||||
throw new Error(result.error ?? 'connection test failed');
|
||||
}
|
||||
return { driver: connector.driver };
|
||||
|
|
@ -325,21 +318,6 @@ async function emitConnectionTest(input: {
|
|||
...(errorDetail ? { errorDetail } : {}),
|
||||
},
|
||||
});
|
||||
if (input.error) {
|
||||
await reportException({
|
||||
error: input.error,
|
||||
context: { source: 'connection test', handled: true, fatal: false },
|
||||
projectDir: input.project.projectDir,
|
||||
io: input.io,
|
||||
redactionSecrets: await collectTelemetryRedactionSecrets({
|
||||
project: input.project,
|
||||
connectionId: input.connectionId,
|
||||
includeLlm: false,
|
||||
includeEmbeddings: false,
|
||||
env: process.env,
|
||||
}),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function visualWidth(text: string): number {
|
||||
|
|
|
|||
|
|
@ -5,9 +5,7 @@ import { assertReadOnlySql, limitSqlForExecution } from '../../context/connectio
|
|||
import { tryConstraintQuery } from '../../context/scan/constraint-discovery.js';
|
||||
import { scopedTableNames } from '../../context/scan/table-ref.js';
|
||||
import {
|
||||
connectorTestFailure,
|
||||
createKtxConnectorCapabilities,
|
||||
type KtxConnectorTestResult,
|
||||
type KtxColumnSampleInput,
|
||||
type KtxColumnSampleResult,
|
||||
type KtxColumnStatsInput,
|
||||
|
|
@ -322,7 +320,7 @@ export class KtxBigQueryScanConnector implements KtxScanConnector {
|
|||
this.id = `bigquery:${options.connectionId}`;
|
||||
}
|
||||
|
||||
async testConnection(): Promise<KtxConnectorTestResult> {
|
||||
async testConnection(): Promise<{ success: boolean; error?: string }> {
|
||||
try {
|
||||
const client = this.getClient();
|
||||
await client.getDatasets({ maxResults: 1 });
|
||||
|
|
@ -331,7 +329,7 @@ export class KtxBigQueryScanConnector implements KtxScanConnector {
|
|||
}
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
return connectorTestFailure(error);
|
||||
return { success: false, error: error instanceof Error ? error.message : String(error) };
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { createClient } from '@clickhouse/client';
|
||||
import { getDialectForDriver } from '../../context/connections/dialects.js';
|
||||
import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js';
|
||||
import { connectorTestFailure, createKtxConnectorCapabilities, type KtxConnectorTestResult, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaColumn, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableRef, type KtxTableSampleInput, type KtxTableListEntry, type KtxTableSampleResult } from '../../context/scan/types.js';
|
||||
import { createKtxConnectorCapabilities, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaColumn, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableRef, type KtxTableSampleInput, type KtxTableListEntry, type KtxTableSampleResult } from '../../context/scan/types.js';
|
||||
import { scopedTableNames } from '../../context/scan/table-ref.js';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { Agent as HttpsAgent } from 'node:https';
|
||||
|
|
@ -317,12 +317,12 @@ export class KtxClickHouseScanConnector implements KtxScanConnector {
|
|||
this.id = `clickhouse:${options.connectionId}`;
|
||||
}
|
||||
|
||||
async testConnection(): Promise<KtxConnectorTestResult> {
|
||||
async testConnection(): Promise<{ success: boolean; error?: string }> {
|
||||
try {
|
||||
await this.query('SELECT 1');
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
return connectorTestFailure(error);
|
||||
return { success: false, error: error instanceof Error ? error.message : String(error) };
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -11,9 +11,7 @@ import {
|
|||
} from '../../context/scan/constraint-discovery.js';
|
||||
import { scopedTableNames } from '../../context/scan/table-ref.js';
|
||||
import {
|
||||
connectorTestFailure,
|
||||
createKtxConnectorCapabilities,
|
||||
type KtxConnectorTestResult,
|
||||
type KtxColumnSampleInput,
|
||||
type KtxColumnSampleResult,
|
||||
type KtxColumnStatsInput,
|
||||
|
|
@ -415,12 +413,12 @@ export class KtxMysqlScanConnector implements KtxScanConnector {
|
|||
this.id = `mysql:${options.connectionId}`;
|
||||
}
|
||||
|
||||
async testConnection(): Promise<KtxConnectorTestResult> {
|
||||
async testConnection(): Promise<{ success: boolean; error?: string }> {
|
||||
try {
|
||||
await this.query('SELECT 1');
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
return connectorTestFailure(error);
|
||||
return { success: false, error: error instanceof Error ? error.message : String(error) };
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -6,9 +6,7 @@ import { assertReadOnlySql, limitSqlForExecution } from '../../context/connectio
|
|||
import { tryConstraintQuery } from '../../context/scan/constraint-discovery.js';
|
||||
import { scopedTableNames } from '../../context/scan/table-ref.js';
|
||||
import {
|
||||
connectorTestFailure,
|
||||
createKtxConnectorCapabilities,
|
||||
type KtxConnectorTestResult,
|
||||
type KtxColumnSampleInput,
|
||||
type KtxColumnSampleResult,
|
||||
type KtxColumnStatsInput,
|
||||
|
|
@ -444,12 +442,12 @@ export class KtxPostgresScanConnector implements KtxScanConnector {
|
|||
this.id = `postgres:${options.connectionId}`;
|
||||
}
|
||||
|
||||
async testConnection(): Promise<KtxConnectorTestResult> {
|
||||
async testConnection(): Promise<{ success: boolean; error?: string }> {
|
||||
try {
|
||||
await this.query('SELECT 1');
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
return connectorTestFailure(error);
|
||||
return { success: false, error: error instanceof Error ? error.message : String(error) };
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -7,9 +7,7 @@ import { assertReadOnlySql, limitSqlForExecution } from '../../context/connectio
|
|||
import { tryConstraintQuery } from '../../context/scan/constraint-discovery.js';
|
||||
import { scopedTableNames } from '../../context/scan/table-ref.js';
|
||||
import {
|
||||
connectorTestFailure,
|
||||
createKtxConnectorCapabilities,
|
||||
type KtxConnectorTestResult,
|
||||
type KtxColumnSampleInput,
|
||||
type KtxColumnSampleResult,
|
||||
type KtxColumnStatsInput,
|
||||
|
|
@ -466,7 +464,7 @@ class SnowflakeSdkDriver implements KtxSnowflakeDriver {
|
|||
await this.query('SELECT 1');
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
return connectorTestFailure(error);
|
||||
return { success: false, error: error instanceof Error ? error.message : String(error) };
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -575,7 +573,7 @@ export class KtxSnowflakeScanConnector implements KtxScanConnector {
|
|||
}
|
||||
}
|
||||
|
||||
async testConnection(): Promise<KtxConnectorTestResult> {
|
||||
async testConnection(): Promise<{ success: boolean; error?: string }> {
|
||||
return this.getDriver().test();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import { fileURLToPath } from 'node:url';
|
|||
import { getDialectForDriver } from '../../context/connections/dialects.js';
|
||||
import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js';
|
||||
import { normalizeQueryRows } from '../../context/connections/query-executor.js';
|
||||
import { connectorTestFailure, createKtxConnectorCapabilities, type KtxConnectorTestResult, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaForeignKey, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableListEntry, type KtxTableRef, type KtxTableSampleInput, type KtxTableSampleResult } from '../../context/scan/types.js';
|
||||
import { createKtxConnectorCapabilities, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaForeignKey, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableListEntry, type KtxTableRef, type KtxTableSampleInput, type KtxTableSampleResult } from '../../context/scan/types.js';
|
||||
import { scopedTableNames } from '../../context/scan/table-ref.js';
|
||||
|
||||
export interface KtxSqliteConnectionConfig {
|
||||
|
|
@ -167,7 +167,7 @@ export class KtxSqliteScanConnector implements KtxScanConnector {
|
|||
this.id = `sqlite:${options.connectionId}`;
|
||||
}
|
||||
|
||||
async testConnection(): Promise<KtxConnectorTestResult> {
|
||||
async testConnection(): Promise<{ success: boolean; error?: string }> {
|
||||
try {
|
||||
if (!existsSync(this.dbPath) || !statSync(this.dbPath).isFile()) {
|
||||
return { success: false, error: `File not found: ${this.dbPath}` };
|
||||
|
|
@ -175,7 +175,7 @@ export class KtxSqliteScanConnector implements KtxScanConnector {
|
|||
this.database().prepare('SELECT 1').get();
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
return connectorTestFailure(error);
|
||||
return { success: false, error: error instanceof Error ? error.message : String(error) };
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -3,9 +3,7 @@ import { getDialectForDriver } from '../../context/connections/dialects.js';
|
|||
import { tryConstraintQuery } from '../../context/scan/constraint-discovery.js';
|
||||
import { scopedTableNames } from '../../context/scan/table-ref.js';
|
||||
import {
|
||||
connectorTestFailure,
|
||||
createKtxConnectorCapabilities,
|
||||
type KtxConnectorTestResult,
|
||||
type KtxColumnSampleInput,
|
||||
type KtxColumnSampleResult,
|
||||
type KtxColumnStatsInput,
|
||||
|
|
@ -386,12 +384,12 @@ export class KtxSqlServerScanConnector implements KtxScanConnector {
|
|||
this.id = `sqlserver:${options.connectionId}`;
|
||||
}
|
||||
|
||||
async testConnection(): Promise<KtxConnectorTestResult> {
|
||||
async testConnection(): Promise<{ success: boolean; error?: string }> {
|
||||
try {
|
||||
await this.query('SELECT 1');
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
return connectorTestFailure(error);
|
||||
return { success: false, error: error instanceof Error ? error.message : String(error) };
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,39 +0,0 @@
|
|||
/** @internal */
|
||||
export function createAbortError(message = 'Aborted'): DOMException {
|
||||
return new DOMException(message, 'AbortError');
|
||||
}
|
||||
|
||||
export function isAbortError(error: unknown): boolean {
|
||||
if (error instanceof DOMException && error.name === 'AbortError') {
|
||||
return true;
|
||||
}
|
||||
if (!error || typeof error !== 'object') {
|
||||
return false;
|
||||
}
|
||||
const record = error as { name?: unknown; code?: unknown };
|
||||
return record.name === 'AbortError' || record.code === 'ABORT_ERR';
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function throwIfAborted(signal?: AbortSignal): void {
|
||||
if (signal?.aborted) {
|
||||
throw createAbortError();
|
||||
}
|
||||
}
|
||||
|
||||
export function linkAbortSignal(parent?: AbortSignal): { controller: AbortController; dispose: () => void } {
|
||||
const controller = new AbortController();
|
||||
if (!parent) {
|
||||
return { controller, dispose: () => undefined };
|
||||
}
|
||||
if (parent.aborted) {
|
||||
controller.abort(createAbortError());
|
||||
return { controller, dispose: () => undefined };
|
||||
}
|
||||
const onAbort = () => controller.abort(createAbortError());
|
||||
parent.addEventListener('abort', onAbort, { once: true });
|
||||
return {
|
||||
controller,
|
||||
dispose: () => parent.removeEventListener('abort', onAbort),
|
||||
};
|
||||
}
|
||||
|
|
@ -23,7 +23,6 @@ export interface QueryHistoryFilterProposal {
|
|||
consideredRoleCount: number;
|
||||
skipped: { reason: 'no-llm' | 'no-daemon' | 'no-in-scope-history' | 'user-block-present' } | null;
|
||||
warnings: string[];
|
||||
parseFailedTemplateIds: string[];
|
||||
}
|
||||
|
||||
export interface ProposeQueryHistoryServiceAccountFiltersInput {
|
||||
|
|
@ -75,7 +74,7 @@ const queryHistoryFilterAdjudicationSchema = z.object({
|
|||
type QueryHistoryFilterAdjudication = z.infer<typeof queryHistoryFilterAdjudicationSchema>;
|
||||
|
||||
function emptyProposal(skipped: QueryHistoryFilterProposal['skipped'], warnings: string[] = []): QueryHistoryFilterProposal {
|
||||
return { excludedRoles: [], consideredRoleCount: 0, skipped, warnings, parseFailedTemplateIds: [] };
|
||||
return { excludedRoles: [], consideredRoleCount: 0, skipped, warnings };
|
||||
}
|
||||
|
||||
function displayTableRef(ref: KtxTableRef): string {
|
||||
|
|
@ -181,7 +180,6 @@ export async function proposeQueryHistoryServiceAccountFilters(
|
|||
const windowDays = 'windowDays' in config ? config.windowDays : 90;
|
||||
const windowStart = new Date(now.getTime() - windowDays * 24 * 60 * 60 * 1000);
|
||||
const warnings: string[] = [];
|
||||
const parseFailedTemplateIds: string[] = [];
|
||||
const snapshot: AggregatedTemplate[] = [];
|
||||
|
||||
try {
|
||||
|
|
@ -214,7 +212,7 @@ export async function proposeQueryHistoryServiceAccountFilters(
|
|||
for (const template of snapshot) {
|
||||
const parsed = analysis.get(template.templateId);
|
||||
if (!parsed || parsed.error) {
|
||||
parseFailedTemplateIds.push(template.templateId);
|
||||
warnings.push(`query_history_filter_picker_parse_failed:${template.templateId}`);
|
||||
continue;
|
||||
}
|
||||
const tablesTouched = [...new Map(parsed.tablesTouched.map((ref) => [tableRefKey(ref), ref])).values()]
|
||||
|
|
@ -238,7 +236,6 @@ export async function proposeQueryHistoryServiceAccountFilters(
|
|||
consideredRoleCount: records.length,
|
||||
skipped: { reason: 'no-in-scope-history' },
|
||||
warnings,
|
||||
parseFailedTemplateIds,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -259,7 +256,6 @@ export async function proposeQueryHistoryServiceAccountFilters(
|
|||
...warnings,
|
||||
`query_history_filter_picker_llm_failed:${error instanceof Error ? error.message : String(error)}`,
|
||||
],
|
||||
parseFailedTemplateIds,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -278,6 +274,5 @@ export async function proposeQueryHistoryServiceAccountFilters(
|
|||
consideredRoleCount: records.length,
|
||||
skipped: input.userServiceAccountsPresent ? { reason: 'user-block-present' } : null,
|
||||
warnings,
|
||||
parseFailedTemplateIds,
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -40,7 +40,6 @@ export interface CuratorPaginationInput {
|
|||
buildToolSet: (passNumber: number) => KtxRuntimeToolSet;
|
||||
getReconciliationActions: () => MemoryAction[];
|
||||
onStepFinish?: (info: { passNumber: number; stepIndex: number; stepBudget: number }) => void;
|
||||
abortSignal?: AbortSignal;
|
||||
}
|
||||
|
||||
interface CuratorPaginationResult extends ReconciliationOutcome {
|
||||
|
|
@ -244,7 +243,6 @@ export class CuratorPaginationService implements CuratorPaginationPort {
|
|||
sourceKey: params.input.sourceKey,
|
||||
jobId: params.input.jobId,
|
||||
forceRun: params.forceRun,
|
||||
abortSignal: params.input.abortSignal,
|
||||
onStepFinish: params.input.onStepFinish
|
||||
? ({ stepIndex, stepBudget }) =>
|
||||
params.input.onStepFinish?.({ passNumber: params.passNumber, stepIndex, stepBudget })
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@ export interface RepairFinalGateFailureInput {
|
|||
repairKind: FinalGateRepairKind;
|
||||
maxAttempts?: number;
|
||||
stepBudget?: number;
|
||||
abortSignal?: AbortSignal;
|
||||
}
|
||||
|
||||
const readRepairFileSchema = z.object({
|
||||
|
|
@ -201,7 +200,6 @@ export async function repairFinalGateFailure(
|
|||
jobId: input.trace.context.jobId,
|
||||
repairKind: input.repairKind,
|
||||
},
|
||||
abortSignal: input.abortSignal,
|
||||
}),
|
||||
);
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ import { dirname, join } from 'node:path';
|
|||
import pLimit from 'p-limit';
|
||||
import { z } from 'zod';
|
||||
import { type KtxLogger, noopLogger } from '../../context/core/config.js';
|
||||
import type { RateLimitWaitState } from '../../context/llm/rate-limit-governor.js';
|
||||
import { createRuntimeToolDescriptorFromAiTool } from '../../context/llm/runtime-tools.js';
|
||||
import type { KtxRuntimeToolSet } from '../../context/llm/runtime-port.js';
|
||||
import type { CaptureSession, MemoryAction } from '../../context/memory/types.js';
|
||||
|
|
@ -220,10 +219,6 @@ export class IngestBundleRunner {
|
|||
}
|
||||
|
||||
async run(job: IngestBundleJob, ctx?: IngestJobContext): Promise<IngestBundleResult> {
|
||||
const unsubscribeRateLimitGovernor = this.subscribeRateLimitGovernor({
|
||||
trace: this.createTrace(job),
|
||||
memoryFlow: ctx?.memoryFlow,
|
||||
});
|
||||
const key = job.connectionId;
|
||||
const previous = this.chainByConnection.get(key);
|
||||
if (previous) {
|
||||
|
|
@ -246,72 +241,10 @@ export class IngestBundleRunner {
|
|||
ctx?.memoryFlow?.finish('error', [sanitizeMemoryFlowError(error)]);
|
||||
throw error;
|
||||
} finally {
|
||||
unsubscribeRateLimitGovernor();
|
||||
await this.maybeEmitIngestProfile(job.jobId);
|
||||
}
|
||||
}
|
||||
|
||||
private formatRateLimitWait(
|
||||
state: Extract<RateLimitWaitState, { kind: 'wait_tick' | 'wait_started' | 'wait_finished' }>,
|
||||
): string {
|
||||
const seconds = Math.ceil(state.remainingMs / 1_000);
|
||||
const minutes = Math.floor(seconds / 60);
|
||||
const remainder = seconds % 60;
|
||||
const duration = minutes > 0 ? `${minutes}m${String(remainder).padStart(2, '0')}s` : `${seconds}s`;
|
||||
const type = state.rateLimitType ? ` ${state.rateLimitType}` : '';
|
||||
return `Rate-limited (${state.provider}${type}); resuming in ${duration}; Ctrl+C to stop`;
|
||||
}
|
||||
|
||||
private subscribeRateLimitGovernor(input: {
|
||||
trace: IngestTraceWriter;
|
||||
memoryFlow?: MemoryFlowEventSink;
|
||||
}): () => void {
|
||||
const governor = this.deps.settings.rateLimitGovernor;
|
||||
if (!governor) {
|
||||
return () => undefined;
|
||||
}
|
||||
return governor.subscribe((state: RateLimitWaitState) => {
|
||||
if (state.kind === 'rate_limit_observed') {
|
||||
void input.trace.event('info', 'rate_limit', 'rate_limit_observed', { ...state });
|
||||
return;
|
||||
}
|
||||
if (state.kind === 'concurrency_adjusted') {
|
||||
void input.trace.event('info', 'rate_limit', 'concurrency_adjusted', { ...state });
|
||||
return;
|
||||
}
|
||||
void input.trace.event('info', 'rate_limit', state.kind, { ...state });
|
||||
if (state.kind === 'wait_tick' || state.kind === 'wait_started') {
|
||||
input.memoryFlow?.emit({
|
||||
type: 'rate_limit_wait',
|
||||
provider: state.provider,
|
||||
...(state.rateLimitType ? { rateLimitType: state.rateLimitType } : {}),
|
||||
resumeAtMs: state.resumeAtMs,
|
||||
remainingMs: state.remainingMs,
|
||||
});
|
||||
input.memoryFlow?.emit({
|
||||
type: 'stage_progress',
|
||||
stage: 'integration',
|
||||
percent: 50,
|
||||
message: this.formatRateLimitWait(state),
|
||||
transient: true,
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private async withRateLimitWorkSlot<T>(abortSignal: AbortSignal | undefined, fn: () => Promise<T>): Promise<T> {
|
||||
const governor = this.deps.settings.rateLimitGovernor;
|
||||
if (!governor) {
|
||||
return fn();
|
||||
}
|
||||
const release = await governor.acquireWorkSlot(abortSignal);
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
release();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* When profiling is enabled — via the `KTX_PROFILE_INGEST` env var or the
|
||||
* `ingest.profile` config setting — read the job's trace + tool transcripts
|
||||
|
|
@ -944,7 +877,6 @@ export class IngestBundleRunner {
|
|||
includeContextEvidenceTools: boolean;
|
||||
currentTableExists(tableRef: string): Promise<boolean>;
|
||||
memoryFlow?: MemoryFlowEventSink;
|
||||
abortSignal?: AbortSignal;
|
||||
wuSkillNames: string[];
|
||||
onStepFinish?: (info: { stepIndex: number; stepBudget: number }) => void;
|
||||
}): Promise<WorkUnitOutcome> {
|
||||
|
|
@ -1097,7 +1029,6 @@ export class IngestBundleRunner {
|
|||
jobId: input.job.jobId,
|
||||
toolFailureCount: (unitKey) => input.transcriptSummaries.get(unitKey)?.fatalErrorCount ?? 0,
|
||||
onStepFinish: input.onStepFinish,
|
||||
abortSignal: input.abortSignal,
|
||||
},
|
||||
input.wu,
|
||||
);
|
||||
|
|
@ -1593,8 +1524,7 @@ export class IngestBundleRunner {
|
|||
try {
|
||||
await Promise.all(
|
||||
workUnits.map((wu, index) =>
|
||||
limitWorkUnit(() =>
|
||||
this.withRateLimitWorkSlot(ctx?.abortSignal, async () => {
|
||||
limitWorkUnit(async () => {
|
||||
const outcome = await runIsolatedWorkUnit({
|
||||
unitIndex: index,
|
||||
ingestionBaseSha,
|
||||
|
|
@ -1602,7 +1532,6 @@ export class IngestBundleRunner {
|
|||
patchDir,
|
||||
trace: runTrace,
|
||||
workUnit: wu,
|
||||
abortSignal: ctx?.abortSignal,
|
||||
afterSuccess: (child) => copyTransientIngestEvidence(child.workdir, sessionWorktree.workdir),
|
||||
run: async (child) => {
|
||||
const scopedWikiService = this.deps.wikiService.forWorktree(child.workdir);
|
||||
|
|
@ -1636,7 +1565,6 @@ export class IngestBundleRunner {
|
|||
includeContextEvidenceTools: adapter.evidenceIndexing === 'documents' && !!contextReport,
|
||||
currentTableExists: (tableRef) =>
|
||||
this.tableRefExistsInSemanticLayer(scopedSemanticLayerService, slConnectionIds, tableRef),
|
||||
abortSignal: ctx?.abortSignal,
|
||||
memoryFlow,
|
||||
wuSkillNames,
|
||||
onStepFinish: ({ stepIndex, stepBudget }) => {
|
||||
|
|
@ -1666,8 +1594,7 @@ export class IngestBundleRunner {
|
|||
completedWorkUnits / workUnits.length,
|
||||
`${completedWorkUnits} of ${workUnits.length} work units complete`,
|
||||
);
|
||||
}),
|
||||
),
|
||||
}),
|
||||
),
|
||||
);
|
||||
} catch (error) {
|
||||
|
|
@ -1766,7 +1693,6 @@ export class IngestBundleRunner {
|
|||
reason: context.reason,
|
||||
maxAttempts: 1,
|
||||
stepBudget: 12,
|
||||
abortSignal: ctx?.abortSignal,
|
||||
});
|
||||
emitStageProgress(
|
||||
'integration',
|
||||
|
|
@ -1788,7 +1714,6 @@ export class IngestBundleRunner {
|
|||
repairKind: 'patch_semantic_gate',
|
||||
maxAttempts: 1,
|
||||
stepBudget: 16,
|
||||
abortSignal: ctx?.abortSignal,
|
||||
});
|
||||
emitStageProgress(
|
||||
'integration',
|
||||
|
|
@ -2068,7 +1993,6 @@ export class IngestBundleRunner {
|
|||
);
|
||||
}
|
||||
: undefined,
|
||||
abortSignal: ctx?.abortSignal,
|
||||
});
|
||||
curatorReport = curatorOutcome.report;
|
||||
curatorWarnings = curatorOutcome.warnings;
|
||||
|
|
@ -2114,7 +2038,6 @@ export class IngestBundleRunner {
|
|||
sourceKey: job.sourceKey,
|
||||
jobId: job.jobId,
|
||||
force: !!overrideReport,
|
||||
abortSignal: ctx?.abortSignal,
|
||||
onStepFinish: stage4
|
||||
? ({ stepIndex, stepBudget }) => {
|
||||
emitStageProgress('reconciliation', 85, `Reconciling results: step ${stepIndex}/${stepBudget}`, {
|
||||
|
|
@ -2547,7 +2470,6 @@ export class IngestBundleRunner {
|
|||
repairKind: 'final_artifact_gate',
|
||||
maxAttempts: 1,
|
||||
stepBudget: 16,
|
||||
abortSignal: ctx?.abortSignal,
|
||||
});
|
||||
|
||||
isolatedDiffSummary.gateRepairAttempts += gateRepair.attempts;
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@ export interface ResolveTextualConflictInput {
|
|||
reason: string;
|
||||
maxAttempts?: number;
|
||||
stepBudget?: number;
|
||||
abortSignal?: AbortSignal;
|
||||
}
|
||||
|
||||
const readIntegrationFileSchema = z.object({
|
||||
|
|
@ -209,7 +208,6 @@ export async function resolveTextualConflict(
|
|||
jobId: input.trace.context.jobId,
|
||||
unitKey: input.unitKey,
|
||||
},
|
||||
abortSignal: input.abortSignal,
|
||||
}),
|
||||
);
|
||||
|
||||
|
|
|
|||
|
|
@ -14,7 +14,6 @@ export interface RunIsolatedWorkUnitInput {
|
|||
patchDir: string;
|
||||
trace: IngestTraceWriter;
|
||||
workUnit: WorkUnit;
|
||||
abortSignal?: AbortSignal;
|
||||
run(child: IngestSessionWorktree): Promise<WorkUnitOutcome>;
|
||||
afterSuccess?(child: IngestSessionWorktree): Promise<void>;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,7 +12,6 @@ import type { KtxSemanticLayerComputePort } from '../../context/daemon/semantic-
|
|||
import { createRuntimeToolDescriptorFromAiTool } from '../../context/llm/runtime-tools.js';
|
||||
import { createLocalKtxLlmRuntimeFromConfig } from '../../context/llm/local-config.js';
|
||||
import { KtxIngestEmbeddingPortAdapter } from '../../context/llm/embedding-port.js';
|
||||
import { createRateLimitGovernorConfig, RateLimitGovernor } from '../../context/llm/rate-limit-governor.js';
|
||||
import { RuntimeAgentRunner, type AgentRunnerPort, type KtxLlmRuntimePort, type KtxRuntimeToolSet } from '../../context/llm/runtime-port.js';
|
||||
import type { KtxEmbeddingProvider } from '../../llm/types.js';
|
||||
import type { KtxLocalProject } from '../../context/project/project.js';
|
||||
|
|
@ -620,7 +619,7 @@ function localIngestLlmProviderGuardMessage(projectDir: string): string {
|
|||
].join('\n');
|
||||
}
|
||||
|
||||
function resolveAgentRunner(options: CreateLocalBundleIngestRuntimeOptions, rateLimitGovernor: RateLimitGovernor): {
|
||||
function resolveAgentRunner(options: CreateLocalBundleIngestRuntimeOptions): {
|
||||
agentRunner: AgentRunnerPort;
|
||||
llmRuntime?: KtxLlmRuntimePort;
|
||||
} {
|
||||
|
|
@ -629,7 +628,6 @@ function resolveAgentRunner(options: CreateLocalBundleIngestRuntimeOptions, rate
|
|||
(options.createLlmRuntime ?? createLocalKtxLlmRuntimeFromConfig)(options.project.config.llm, {
|
||||
projectDir: options.project.projectDir,
|
||||
env: process.env,
|
||||
rateLimitGovernor,
|
||||
}) ??
|
||||
undefined;
|
||||
|
||||
|
|
@ -679,13 +677,7 @@ export function createLocalBundleIngestRuntime(
|
|||
const knowledgeIndex = new LocalKnowledgeIndex(options.project, embedding);
|
||||
const knowledgeEvents = new NoopKnowledgeEventPort();
|
||||
const wikiService = new KnowledgeWikiService(rootFileStore, embedding, knowledgeIndex, options.project.git, logger);
|
||||
const rateLimitGovernor = new RateLimitGovernor(
|
||||
createRateLimitGovernorConfig({
|
||||
...options.project.config.ingest.rateLimit,
|
||||
maxConcurrency: options.project.config.ingest.workUnits.maxConcurrency,
|
||||
}),
|
||||
);
|
||||
const { agentRunner, llmRuntime } = resolveAgentRunner(options, rateLimitGovernor);
|
||||
const { agentRunner, llmRuntime } = resolveAgentRunner(options);
|
||||
const promptService = new PromptService({ promptsDir, partials: [], logger });
|
||||
const storage = new LocalIngestStorage(options.project);
|
||||
const registry = registerAdapters(options.adapters);
|
||||
|
|
@ -725,7 +717,6 @@ export function createLocalBundleIngestRuntime(
|
|||
workUnitMaxConcurrency: options.project.config.ingest.workUnits.maxConcurrency,
|
||||
workUnitStepBudget: options.project.config.ingest.workUnits.stepBudget,
|
||||
workUnitFailureMode: options.project.config.ingest.workUnits.failureMode,
|
||||
rateLimitGovernor,
|
||||
profileIngest: options.project.config.ingest.profile,
|
||||
ingestTraceLevel: ingestTraceLevelFromEnv(),
|
||||
},
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ import { cp, mkdir, rm } from 'node:fs/promises';
|
|||
import { isAbsolute, resolve } from 'node:path';
|
||||
import type { KtxSqlQueryExecutorPort } from '../../context/connections/query-executor.js';
|
||||
import type { KtxLogger } from '../../context/core/config.js';
|
||||
import { createAbortError, isAbortError } from '../../context/core/abort.js';
|
||||
import type { KtxSemanticLayerComputePort } from '../../context/daemon/semantic-layer-compute.js';
|
||||
import type { AgentRunnerPort, KtxLlmRuntimePort } from '../../context/llm/runtime-port.js';
|
||||
import type { KtxLocalProject } from '../../context/project/project.js';
|
||||
|
|
@ -37,7 +36,6 @@ export interface RunLocalIngestOptions {
|
|||
queryExecutor?: KtxSqlQueryExecutorPort;
|
||||
logger?: KtxLogger;
|
||||
embeddingProvider?: import('../../llm/types.js').KtxEmbeddingProvider | null;
|
||||
abortSignal?: AbortSignal;
|
||||
}
|
||||
|
||||
export interface LocalIngestResult {
|
||||
|
|
@ -125,11 +123,10 @@ function findAdapter(adapters: SourceAdapter[], source: string): SourceAdapter {
|
|||
return adapter;
|
||||
}
|
||||
|
||||
function localJobContext(jobId: string, memoryFlow?: MemoryFlowEventSink, abortSignal?: AbortSignal): IngestJobContext {
|
||||
function localJobContext(jobId: string, memoryFlow?: MemoryFlowEventSink): IngestJobContext {
|
||||
return {
|
||||
jobId,
|
||||
...(memoryFlow ? { memoryFlow } : {}),
|
||||
...(abortSignal ? { abortSignal } : {}),
|
||||
startPhase() {
|
||||
return new LocalIngestPhase();
|
||||
},
|
||||
|
|
@ -161,7 +158,6 @@ async function runScheduledPullJob(options: {
|
|||
queryExecutor?: KtxSqlQueryExecutorPort;
|
||||
logger?: KtxLogger;
|
||||
embeddingProvider?: import('../../llm/types.js').KtxEmbeddingProvider | null;
|
||||
abortSignal?: AbortSignal;
|
||||
}): Promise<LocalIngestResult> {
|
||||
const runtime = createLocalBundleIngestRuntime(options);
|
||||
const jobId = options.jobId ?? runtime.nextJobId();
|
||||
|
|
@ -173,7 +169,7 @@ async function runScheduledPullJob(options: {
|
|||
trigger: options.trigger ?? 'manual_resync',
|
||||
bundleRef: { kind: 'scheduled_pull', config: options.pullConfig },
|
||||
},
|
||||
localJobContext(jobId, options.memoryFlow, options.abortSignal),
|
||||
localJobContext(jobId, options.memoryFlow),
|
||||
);
|
||||
const report = await runtime.store.findByJobId(jobId);
|
||||
if (!report) {
|
||||
|
|
@ -216,7 +212,6 @@ export async function runLocalIngest(options: RunLocalIngestOptions): Promise<Lo
|
|||
queryExecutor: options.queryExecutor,
|
||||
logger: options.logger,
|
||||
embeddingProvider: options.embeddingProvider,
|
||||
abortSignal: options.abortSignal,
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -228,7 +223,7 @@ export async function runLocalIngest(options: RunLocalIngestOptions): Promise<Lo
|
|||
trigger: options.trigger ?? (options.sourceDir ? 'upload' : 'manual_resync'),
|
||||
bundleRef,
|
||||
},
|
||||
localJobContext(jobId, options.memoryFlow, options.abortSignal),
|
||||
localJobContext(jobId, options.memoryFlow),
|
||||
);
|
||||
const report = await runtime.store.findByJobId(jobId);
|
||||
if (!report) {
|
||||
|
|
@ -367,9 +362,6 @@ export async function runLocalMetabaseIngest(
|
|||
|
||||
const children: LocalMetabaseFanoutChild[] = [];
|
||||
for (const childPlan of childPlans) {
|
||||
if (options.abortSignal?.aborted) {
|
||||
throw createAbortError();
|
||||
}
|
||||
const targetConnectionId = safeSegment('target connection id', childPlan.targetConnectionId);
|
||||
if (!options.project.config.connections[targetConnectionId]) {
|
||||
throw new Error(`Target connection "${targetConnectionId}" is not configured in ktx.yaml`);
|
||||
|
|
@ -399,12 +391,8 @@ export async function runLocalMetabaseIngest(
|
|||
queryExecutor: options.queryExecutor,
|
||||
logger: options.logger,
|
||||
embeddingProvider: options.embeddingProvider,
|
||||
abortSignal: options.abortSignal,
|
||||
});
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) {
|
||||
throw error;
|
||||
}
|
||||
child = await recordLocalMetabaseChildFailure({
|
||||
project: options.project,
|
||||
jobId: childJobId,
|
||||
|
|
|
|||
|
|
@ -70,13 +70,6 @@ const memoryFlowEventSchema = z.discriminatedUnion('type', [
|
|||
message: z.string().min(1),
|
||||
transient: z.boolean().optional(),
|
||||
}),
|
||||
eventSchema({
|
||||
type: z.literal('rate_limit_wait'),
|
||||
provider: z.string(),
|
||||
rateLimitType: z.string().optional(),
|
||||
resumeAtMs: z.number().int().nonnegative(),
|
||||
remainingMs: z.number().int().nonnegative(),
|
||||
}),
|
||||
eventSchema({
|
||||
type: z.literal('work_unit_started'),
|
||||
unitKey: z.string().min(1),
|
||||
|
|
|
|||
|
|
@ -60,13 +60,6 @@ type MemoryFlowEventPayload =
|
|||
message: string;
|
||||
transient?: boolean;
|
||||
}
|
||||
| {
|
||||
type: 'rate_limit_wait';
|
||||
provider: string;
|
||||
rateLimitType?: string;
|
||||
resumeAtMs: number;
|
||||
remainingMs: number;
|
||||
}
|
||||
| {
|
||||
type: 'work_unit_started';
|
||||
unitKey: string;
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ import type { KtxFileStorePort } from '../../context/core/file-store.js';
|
|||
import type { KtxLogger } from '../../context/core/config.js';
|
||||
import type { SessionOutcome } from '../../context/core/session-worktree.service.js';
|
||||
import type { AgentRunnerPort, KtxLlmRuntimePort, KtxRuntimeToolSet } from '../../context/llm/runtime-port.js';
|
||||
import type { RateLimitGovernor } from '../llm/rate-limit-governor.js';
|
||||
import type { MemoryAction, MemoryKnowledgeSlRefsPort } from '../../context/memory/types.js';
|
||||
import type { PromptService } from '../../context/prompts/prompt.service.js';
|
||||
import type { SkillsRegistryService } from '../../context/skills/skills-registry.service.js';
|
||||
|
|
@ -145,7 +144,6 @@ interface IngestSettingsPort {
|
|||
workUnitMaxConcurrency?: number;
|
||||
workUnitStepBudget?: number;
|
||||
workUnitFailureMode?: 'abort' | 'continue';
|
||||
rateLimitGovernor?: RateLimitGovernor;
|
||||
/** Print a timing breakdown to stderr at the end of each run (config-driven; see also KTX_PROFILE_INGEST). `'json'` emits the raw structured profile. */
|
||||
profileIngest?: boolean | 'json';
|
||||
ingestTraceLevel?: IngestTraceLevel;
|
||||
|
|
@ -325,7 +323,6 @@ export interface CuratorPaginationPort {
|
|||
buildToolSet: (passNumber: number) => KtxRuntimeToolSet;
|
||||
getReconciliationActions: () => MemoryAction[];
|
||||
onStepFinish?: (info: { passNumber: number; stepIndex: number; stepBudget: number }) => void;
|
||||
abortSignal?: AbortSignal;
|
||||
}): Promise<ReconciliationOutcome & { report: CuratorPaginationReport; warnings: string[] }>;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
import type { KtxModelRole } from '../../../llm/types.js';
|
||||
import { isAbortError } from '../../core/abort.js';
|
||||
import type { AgentRunnerPort, KtxRuntimeToolSet, RunLoopMetrics } from '../../../context/llm/runtime-port.js';
|
||||
import type { CaptureSession, MemoryAction } from '../../../context/memory/types.js';
|
||||
import { listTouchedSlSources, type TouchedSlSource } from '../../../context/tools/touched-sl-sources.js';
|
||||
|
|
@ -29,7 +28,6 @@ export interface WorkUnitExecutionDeps {
|
|||
connectionId: string;
|
||||
jobId: string;
|
||||
onStepFinish?: (info: { stepIndex: number; stepBudget: number }) => void;
|
||||
abortSignal?: AbortSignal;
|
||||
toolFailureCount?: (unitKey: string) => number;
|
||||
}
|
||||
|
||||
|
|
@ -108,12 +106,8 @@ export async function executeWorkUnit(deps: WorkUnitExecutionDeps, wu: WorkUnit)
|
|||
jobId: deps.jobId,
|
||||
},
|
||||
onStepFinish: deps.onStepFinish,
|
||||
abortSignal: deps.abortSignal,
|
||||
});
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) {
|
||||
throw error;
|
||||
}
|
||||
return failWithResetFromCurrentHead(error instanceof Error ? error.message : String(error));
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -16,7 +16,6 @@ export interface ReconciliationContext {
|
|||
jobId: string;
|
||||
force?: boolean;
|
||||
onStepFinish?: (info: { stepIndex: number; stepBudget: number }) => void;
|
||||
abortSignal?: AbortSignal;
|
||||
forceRun?: boolean;
|
||||
}
|
||||
|
||||
|
|
@ -41,7 +40,6 @@ export async function runReconciliationStage4(ctx: ReconciliationContext): Promi
|
|||
stepBudget: ctx.stepBudget,
|
||||
telemetryTags: { operationName: 'ingest-bundle-reconcile', source: ctx.sourceKey, jobId: ctx.jobId },
|
||||
onStepFinish: ctx.onStepFinish,
|
||||
abortSignal: ctx.abortSignal,
|
||||
});
|
||||
return { skipped: false, stopReason: run.stopReason, error: run.error, ...(run.metrics ? { metrics: run.metrics } : {}) };
|
||||
}
|
||||
|
|
|
|||
|
|
@ -220,6 +220,5 @@ export interface IngestJobPhase {
|
|||
export interface IngestJobContext {
|
||||
jobId: string;
|
||||
memoryFlow?: MemoryFlowEventSink;
|
||||
abortSignal?: AbortSignal;
|
||||
startPhase(weight: number): IngestJobPhase;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,9 +3,7 @@ import type { KtxLlmProvider } from '../../llm/types.js';
|
|||
import { generateText, Output, stepCountIs, type FlexibleSchema, type TelemetrySettings, type ToolSet } from 'ai';
|
||||
import type { z } from 'zod';
|
||||
import { noopLogger, type KtxLogger } from '../../context/core/config.js';
|
||||
import { isAbortError } from '../core/abort.js';
|
||||
import { summarizeKtxLlmDebugRequest, type KtxLlmDebugRequestRecorder } from './debug-request-recorder.js';
|
||||
import type { RateLimitGovernor, RateLimitProvider, RateLimitSignal } from './rate-limit-governor.js';
|
||||
import { createAiSdkToolSet } from './runtime-tools.js';
|
||||
import type {
|
||||
KtxGenerateObjectInput,
|
||||
|
|
@ -42,129 +40,12 @@ export interface AiSdkKtxLlmRuntimeDeps {
|
|||
telemetry?: AgentTelemetryPort;
|
||||
logger?: KtxLogger;
|
||||
debugRequestRecorder?: KtxLlmDebugRequestRecorder;
|
||||
rateLimitGovernor?: Pick<RateLimitGovernor, 'waitForReady' | 'report' | 'maxRetryAttempts'>;
|
||||
}
|
||||
|
||||
function hasTools(tools: Record<string, unknown>): boolean {
|
||||
return Object.keys(tools).length > 0;
|
||||
}
|
||||
|
||||
function modelProviderName(model: unknown): RateLimitProvider {
|
||||
const provider = (model as { provider?: string }).provider ?? '';
|
||||
return provider.includes('vertex') || provider.includes('google') ? 'vertex' : 'anthropic-api';
|
||||
}
|
||||
|
||||
interface HeaderLimitPair {
|
||||
limit: string;
|
||||
remaining: string;
|
||||
rateLimitType: string;
|
||||
}
|
||||
|
||||
const RATE_LIMIT_HEADER_PAIRS: HeaderLimitPair[] = [
|
||||
{
|
||||
limit: 'anthropic-ratelimit-requests-limit',
|
||||
remaining: 'anthropic-ratelimit-requests-remaining',
|
||||
rateLimitType: 'rpm',
|
||||
},
|
||||
{
|
||||
limit: 'anthropic-ratelimit-tokens-limit',
|
||||
remaining: 'anthropic-ratelimit-tokens-remaining',
|
||||
rateLimitType: 'tpm',
|
||||
},
|
||||
{
|
||||
limit: 'anthropic-ratelimit-input-tokens-limit',
|
||||
remaining: 'anthropic-ratelimit-input-tokens-remaining',
|
||||
rateLimitType: 'itpm',
|
||||
},
|
||||
{
|
||||
limit: 'anthropic-ratelimit-output-tokens-limit',
|
||||
remaining: 'anthropic-ratelimit-output-tokens-remaining',
|
||||
rateLimitType: 'otpm',
|
||||
},
|
||||
{
|
||||
limit: 'x-ratelimit-limit-requests',
|
||||
remaining: 'x-ratelimit-remaining-requests',
|
||||
rateLimitType: 'rpm',
|
||||
},
|
||||
{
|
||||
limit: 'x-ratelimit-limit-tokens',
|
||||
remaining: 'x-ratelimit-remaining-tokens',
|
||||
rateLimitType: 'tpm',
|
||||
},
|
||||
];
|
||||
|
||||
function normalizeHeaders(headers: unknown): Record<string, string> {
|
||||
if (!headers || typeof headers !== 'object') {
|
||||
return {};
|
||||
}
|
||||
const get = (headers as { get?: unknown }).get;
|
||||
if (typeof get === 'function') {
|
||||
const out: Record<string, string> = {};
|
||||
for (const pair of RATE_LIMIT_HEADER_PAIRS) {
|
||||
const limit = get.call(headers, pair.limit);
|
||||
const remaining = get.call(headers, pair.remaining);
|
||||
if (typeof limit === 'string') out[pair.limit] = limit;
|
||||
if (typeof remaining === 'string') out[pair.remaining] = remaining;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
return Object.fromEntries(
|
||||
Object.entries(headers as Record<string, unknown>)
|
||||
.filter((entry): entry is [string, string | number] => typeof entry[1] === 'string' || typeof entry[1] === 'number')
|
||||
.map(([key, value]) => [key.toLowerCase(), String(value)]),
|
||||
);
|
||||
}
|
||||
|
||||
function numericHeader(headers: Record<string, string>, key: string): number | undefined {
|
||||
const value = Number(headers[key]);
|
||||
return Number.isFinite(value) && value >= 0 ? value : undefined;
|
||||
}
|
||||
|
||||
function utilizationForPair(headers: Record<string, string>, pair: HeaderLimitPair): number | undefined {
|
||||
const limit = numericHeader(headers, pair.limit);
|
||||
const remaining = numericHeader(headers, pair.remaining);
|
||||
if (limit === undefined || remaining === undefined || limit <= 0) {
|
||||
return undefined;
|
||||
}
|
||||
return 1 - Math.min(limit, remaining) / limit;
|
||||
}
|
||||
|
||||
function aiSdkHeaderRateLimitSignal(provider: RateLimitProvider, result: unknown): RateLimitSignal | undefined {
|
||||
const headers = normalizeHeaders((result as { response?: { headers?: unknown } }).response?.headers);
|
||||
let best: { utilization: number; rateLimitType: string } | undefined;
|
||||
for (const pair of RATE_LIMIT_HEADER_PAIRS) {
|
||||
const utilization = utilizationForPair(headers, pair);
|
||||
if (utilization === undefined) {
|
||||
continue;
|
||||
}
|
||||
if (!best || utilization > best.utilization) {
|
||||
best = { utilization, rateLimitType: pair.rateLimitType };
|
||||
}
|
||||
}
|
||||
if (!best) {
|
||||
return undefined;
|
||||
}
|
||||
return {
|
||||
provider,
|
||||
status: 'allowed',
|
||||
rateLimitType: best.rateLimitType,
|
||||
utilization: Number(best.utilization.toFixed(4)),
|
||||
};
|
||||
}
|
||||
|
||||
function retryAfterMs(error: unknown): number | undefined {
|
||||
const value = (error as { retryAfter?: unknown }).retryAfter;
|
||||
if (typeof value === 'number' && Number.isFinite(value) && value > 0) {
|
||||
return value < 1_000 ? value * 1_000 : value;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function isAiSdkRateLimitError(error: unknown): boolean {
|
||||
const record = error as { name?: string; statusCode?: number; status?: number };
|
||||
return record.name === 'TooManyRequestsError' || record.statusCode === 429 || record.status === 429;
|
||||
}
|
||||
|
||||
export class AiSdkKtxLlmRuntime implements KtxLlmRuntimePort {
|
||||
private readonly logger: KtxLogger;
|
||||
|
||||
|
|
@ -172,41 +53,6 @@ export class AiSdkKtxLlmRuntime implements KtxLlmRuntimePort {
|
|||
this.logger = deps.logger ?? noopLogger;
|
||||
}
|
||||
|
||||
private async generateTextWithRateLimitRetry<T>(
|
||||
provider: RateLimitProvider,
|
||||
abortSignal: AbortSignal | undefined,
|
||||
run: () => Promise<T>,
|
||||
): Promise<T> {
|
||||
// maxRetryAttempts() returns 1 when no governor is present or pacing is
|
||||
// disabled, so a 429 throws immediately instead of hammering the provider
|
||||
// with no backoff; the AI SDK's own maxRetries still handles transient 429s.
|
||||
const maxAttempts = this.deps.rateLimitGovernor?.maxRetryAttempts() ?? 1;
|
||||
let attempt = 0;
|
||||
while (true) {
|
||||
await this.deps.rateLimitGovernor?.waitForReady(abortSignal);
|
||||
try {
|
||||
const result = await run();
|
||||
const signal = aiSdkHeaderRateLimitSignal(provider, result);
|
||||
if (signal) {
|
||||
this.deps.rateLimitGovernor?.report(signal);
|
||||
}
|
||||
return result;
|
||||
} catch (error) {
|
||||
if (isAbortError(error) || !isAiSdkRateLimitError(error) || attempt >= maxAttempts - 1) {
|
||||
throw error;
|
||||
}
|
||||
attempt += 1;
|
||||
const retryAfter = retryAfterMs(error);
|
||||
this.deps.rateLimitGovernor?.report({
|
||||
provider,
|
||||
status: 'rejected',
|
||||
rateLimitType: 'http_429',
|
||||
...(retryAfter !== undefined ? { retryAfterMs: retryAfter } : {}),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async generateText(input: KtxGenerateTextInput): Promise<string> {
|
||||
const model = this.deps.llmProvider.getModel(input.role);
|
||||
if ((model as { provider?: string }).provider === 'deterministic') {
|
||||
|
|
@ -221,13 +67,12 @@ export class AiSdkKtxLlmRuntime implements KtxLlmRuntimePort {
|
|||
});
|
||||
const split = splitKtxSystemMessages(built.messages);
|
||||
const startedAt = Date.now();
|
||||
const request = {
|
||||
const result = await generateText({
|
||||
model,
|
||||
temperature: input.temperature ?? 0,
|
||||
...(split.system ? { system: split.system } : {}),
|
||||
messages: split.messages,
|
||||
tools: built.tools as ToolSet,
|
||||
...(input.abortSignal ? { abortSignal: input.abortSignal } : {}),
|
||||
...(hasTools(tools)
|
||||
? {
|
||||
experimental_repairToolCall: this.deps.llmProvider.repairToolCallHandler({
|
||||
|
|
@ -235,8 +80,7 @@ export class AiSdkKtxLlmRuntime implements KtxLlmRuntimePort {
|
|||
}),
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
const result = await this.generateTextWithRateLimitRetry(modelProviderName(model), input.abortSignal, () => generateText(request));
|
||||
});
|
||||
input.onMetrics?.({ totalMs: Date.now() - startedAt, usage: toLlmTokenUsage(result.totalUsage ?? result.usage) });
|
||||
if (typeof result.text !== 'string') {
|
||||
throw new Error('KTX LLM text generation returned no text');
|
||||
|
|
@ -257,13 +101,12 @@ export class AiSdkKtxLlmRuntime implements KtxLlmRuntimePort {
|
|||
});
|
||||
const split = splitKtxSystemMessages(built.messages);
|
||||
const startedAt = Date.now();
|
||||
const request = {
|
||||
const result = await generateText({
|
||||
model,
|
||||
temperature: input.temperature ?? 0,
|
||||
...(split.system ? { system: split.system } : {}),
|
||||
messages: split.messages,
|
||||
tools: built.tools as ToolSet,
|
||||
...(input.abortSignal ? { abortSignal: input.abortSignal } : {}),
|
||||
...(hasTools(tools)
|
||||
? {
|
||||
experimental_repairToolCall: this.deps.llmProvider.repairToolCallHandler({
|
||||
|
|
@ -272,8 +115,7 @@ export class AiSdkKtxLlmRuntime implements KtxLlmRuntimePort {
|
|||
}
|
||||
: {}),
|
||||
output: Output.object({ schema: input.schema as unknown as FlexibleSchema<TOutput> }),
|
||||
};
|
||||
const result = await this.generateTextWithRateLimitRetry(modelProviderName(model), input.abortSignal, () => generateText(request));
|
||||
});
|
||||
input.onMetrics?.({ totalMs: Date.now() - startedAt, usage: toLlmTokenUsage(result.totalUsage ?? result.usage) });
|
||||
if (result.output == null) {
|
||||
throw new Error('KTX LLM object generation returned no output');
|
||||
|
|
@ -310,7 +152,7 @@ export class AiSdkKtxLlmRuntime implements KtxLlmRuntimePort {
|
|||
}),
|
||||
);
|
||||
|
||||
const request = {
|
||||
const result = await generateText({
|
||||
model,
|
||||
temperature: 0,
|
||||
stopWhen: stepCountIs(params.stepBudget),
|
||||
|
|
@ -321,7 +163,6 @@ export class AiSdkKtxLlmRuntime implements KtxLlmRuntimePort {
|
|||
...(promptMessages.system ? { system: promptMessages.system } : {}),
|
||||
messages: promptMessages.messages,
|
||||
tools: built.tools as ToolSet,
|
||||
...(params.abortSignal ? { abortSignal: params.abortSignal } : {}),
|
||||
onStepFinish: async () => {
|
||||
stepIndex += 1;
|
||||
stepBoundariesMs.push(Date.now() - startedAt);
|
||||
|
|
@ -338,8 +179,7 @@ export class AiSdkKtxLlmRuntime implements KtxLlmRuntimePort {
|
|||
);
|
||||
}
|
||||
},
|
||||
};
|
||||
const result = await this.generateTextWithRateLimitRetry(modelProviderName(model), params.abortSignal, () => generateText(request));
|
||||
});
|
||||
return {
|
||||
stopReason: 'natural',
|
||||
metrics: {
|
||||
|
|
@ -350,9 +190,6 @@ export class AiSdkKtxLlmRuntime implements KtxLlmRuntimePort {
|
|||
},
|
||||
};
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) {
|
||||
throw error;
|
||||
}
|
||||
const err = error instanceof Error ? error : new Error(String(error));
|
||||
this.logger.warn(`[agent-runner] loop failed: ${err.message}`);
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -7,10 +7,8 @@ import {
|
|||
} from '@anthropic-ai/claude-agent-sdk';
|
||||
import { z } from 'zod';
|
||||
import { noopLogger, type KtxLogger } from '../../context/core/config.js';
|
||||
import { createAbortError, isAbortError, throwIfAborted } from '../core/abort.js';
|
||||
import { createKtxClaudeCodeEnv } from './claude-code-env.js';
|
||||
import { resolveClaudeCodeModel } from './claude-code-models.js';
|
||||
import type { RateLimitGovernor, RateLimitSignal } from './rate-limit-governor.js';
|
||||
import { createClaudeSdkTools, mcpToolIds } from './runtime-tools.js';
|
||||
import type {
|
||||
KtxGenerateObjectInput,
|
||||
|
|
@ -23,16 +21,7 @@ import type {
|
|||
RunLoopStopReason,
|
||||
} from './runtime-port.js';
|
||||
|
||||
type QueryResult = AsyncIterable<SDKMessage> & {
|
||||
interrupt?: () => void | Promise<void>;
|
||||
};
|
||||
|
||||
type QueryFn = (params: Parameters<typeof defaultQuery>[0]) => QueryResult;
|
||||
|
||||
interface ClaudeQueryOutcome {
|
||||
result: SDKResultMessage;
|
||||
rejectedRateLimitSignal?: RateLimitSignal;
|
||||
}
|
||||
type QueryFn = (params: Parameters<typeof defaultQuery>[0]) => AsyncIterable<SDKMessage>;
|
||||
|
||||
function claudeTokenUsage(result: SDKResultMessage): LlmTokenUsage {
|
||||
const usage = (result as { usage?: { input_tokens?: number; output_tokens?: number } }).usage;
|
||||
|
|
@ -54,7 +43,6 @@ export interface ClaudeCodeKtxLlmRuntimeDeps {
|
|||
query?: QueryFn;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
logger?: KtxLogger;
|
||||
rateLimitGovernor?: Pick<RateLimitGovernor, 'waitForReady' | 'report' | 'maxRetryAttempts'>;
|
||||
}
|
||||
|
||||
const BUILTIN_TOOLS = [
|
||||
|
|
@ -169,74 +157,6 @@ function expectedMcpServerNames(tools: KtxRuntimeToolSet | undefined): Set<strin
|
|||
return tools && Object.keys(tools).length > 0 ? new Set([KTX_MCP_SERVER_NAME]) : new Set();
|
||||
}
|
||||
|
||||
const CLAUDE_RATE_LIMIT_ERROR_MARKERS = /\b429\b|rate limit|too many requests|quota exceeded|overloaded|max_retries/i;
|
||||
|
||||
function normalizeClaudeResetAtMs(value: unknown): number | undefined {
|
||||
if (typeof value === 'number' && Number.isFinite(value) && value > 0) {
|
||||
return Math.round(value < 10_000_000_000 ? value * 1_000 : value);
|
||||
}
|
||||
if (typeof value === 'string') {
|
||||
const numeric = Number(value);
|
||||
if (Number.isFinite(numeric) && numeric > 0) {
|
||||
return normalizeClaudeResetAtMs(numeric);
|
||||
}
|
||||
const parsed = Date.parse(value);
|
||||
return Number.isFinite(parsed) ? parsed : undefined;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function isClaudeRateLimitResult(result: SDKResultMessage, rejectedSignal: RateLimitSignal | undefined): boolean {
|
||||
const error = resultError(result);
|
||||
if (!error) {
|
||||
return false;
|
||||
}
|
||||
if (rejectedSignal?.status === 'rejected') {
|
||||
return true;
|
||||
}
|
||||
const resultDetails = result as {
|
||||
stop_reason?: unknown;
|
||||
terminal_reason?: unknown;
|
||||
errors?: unknown[];
|
||||
};
|
||||
const details = [
|
||||
error.message,
|
||||
resultDetails.stop_reason,
|
||||
resultDetails.terminal_reason,
|
||||
...(resultDetails.errors ?? []),
|
||||
]
|
||||
.filter((value): value is string => typeof value === 'string' && value.length > 0)
|
||||
.join('\n');
|
||||
return CLAUDE_RATE_LIMIT_ERROR_MARKERS.test(details);
|
||||
}
|
||||
|
||||
function claudeRateLimitSignal(message: SDKMessage): RateLimitSignal | null {
|
||||
const record = message as unknown as Record<string, unknown>;
|
||||
if (record.type === 'rate_limit_event') {
|
||||
const info = record.rate_limit_info as Record<string, unknown> | undefined;
|
||||
if (!info) return null;
|
||||
const rawStatus = typeof info.status === 'string' ? info.status : 'allowed';
|
||||
const resetAtMs = normalizeClaudeResetAtMs(info.resetsAt);
|
||||
return {
|
||||
provider: 'claude-subscription',
|
||||
status: rawStatus === 'rejected' ? 'rejected' : rawStatus === 'allowed_warning' ? 'warning' : 'allowed',
|
||||
...(resetAtMs !== undefined ? { resetAtMs } : {}),
|
||||
...(typeof info.rateLimitType === 'string' ? { rateLimitType: info.rateLimitType } : {}),
|
||||
...(typeof info.utilization === 'number' ? { utilization: info.utilization } : {}),
|
||||
};
|
||||
}
|
||||
if (record.subtype === 'api_retry' || record.type === 'api_retry') {
|
||||
const retryDelayMs = typeof record.retry_delay_ms === 'number' ? record.retry_delay_ms : undefined;
|
||||
return {
|
||||
provider: 'claude-subscription',
|
||||
status: 'warning',
|
||||
...(retryDelayMs !== undefined ? { retryAfterMs: retryDelayMs } : {}),
|
||||
rateLimitType: 'api_retry',
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function managedMcpSettings(serverNames: string[]): NonNullable<Options['managedSettings']> {
|
||||
return {
|
||||
allowManagedMcpServersOnly: true,
|
||||
|
|
@ -297,63 +217,21 @@ async function collectResult(params: {
|
|||
allowedToolIds: Set<string>;
|
||||
expectedMcpServerNames: Set<string>;
|
||||
onAssistantTurn?: () => Promise<void>;
|
||||
rateLimitGovernor?: Pick<RateLimitGovernor, 'waitForReady' | 'report' | 'maxRetryAttempts'>;
|
||||
abortSignal?: AbortSignal;
|
||||
}): Promise<ClaudeQueryOutcome> {
|
||||
}): Promise<SDKResultMessage> {
|
||||
let result: SDKResultMessage | undefined;
|
||||
let rejectedRateLimitSignal: RateLimitSignal | undefined;
|
||||
throwIfAborted(params.abortSignal);
|
||||
await params.rateLimitGovernor?.waitForReady(params.abortSignal);
|
||||
throwIfAborted(params.abortSignal);
|
||||
const queryResult = params.query({ prompt: params.prompt, options: params.options });
|
||||
const onAbort = () => {
|
||||
void Promise.resolve(queryResult.interrupt?.()).catch(() => undefined);
|
||||
};
|
||||
params.abortSignal?.addEventListener('abort', onAbort, { once: true });
|
||||
try {
|
||||
for await (const message of queryResult) {
|
||||
throwIfAborted(params.abortSignal);
|
||||
const rateLimitSignal = claudeRateLimitSignal(message);
|
||||
if (rateLimitSignal) {
|
||||
if (rateLimitSignal.status === 'rejected') {
|
||||
rejectedRateLimitSignal = rateLimitSignal;
|
||||
}
|
||||
params.rateLimitGovernor?.report(rateLimitSignal);
|
||||
}
|
||||
assertInitIsolation(message, params.allowedToolIds, params.expectedMcpServerNames);
|
||||
if (countsAsAssistantTurn(message)) {
|
||||
await params.onAssistantTurn?.();
|
||||
}
|
||||
if (isResult(message)) {
|
||||
result = message;
|
||||
}
|
||||
for await (const message of params.query({ prompt: params.prompt, options: params.options })) {
|
||||
assertInitIsolation(message, params.allowedToolIds, params.expectedMcpServerNames);
|
||||
if (countsAsAssistantTurn(message)) {
|
||||
await params.onAssistantTurn?.();
|
||||
}
|
||||
if (isResult(message)) {
|
||||
result = message;
|
||||
}
|
||||
} finally {
|
||||
params.abortSignal?.removeEventListener('abort', onAbort);
|
||||
}
|
||||
if (params.abortSignal?.aborted) {
|
||||
throw createAbortError();
|
||||
}
|
||||
if (!result) {
|
||||
throw new Error('Claude Code query returned no result message');
|
||||
}
|
||||
return {
|
||||
result,
|
||||
...(rejectedRateLimitSignal ? { rejectedRateLimitSignal } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
async function collectResultWithRateLimitRetry(params: Parameters<typeof collectResult>[0]): Promise<SDKResultMessage> {
|
||||
// maxRetryAttempts() returns 1 when no governor is present or pacing is
|
||||
// disabled, so a rate-limited result surfaces without an extra query; the
|
||||
// Claude Code SDK applies its own backoff for transient rejections.
|
||||
const maxAttempts = params.rateLimitGovernor?.maxRetryAttempts() ?? 1;
|
||||
for (let attempt = 0; ; attempt += 1) {
|
||||
const outcome = await collectResult(params);
|
||||
if (!isClaudeRateLimitResult(outcome.result, outcome.rejectedRateLimitSignal) || attempt >= maxAttempts - 1) {
|
||||
return outcome.result;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export class ClaudeCodeKtxLlmRuntime implements KtxLlmRuntimePort {
|
||||
|
|
@ -374,14 +252,12 @@ export class ClaudeCodeKtxLlmRuntime implements KtxLlmRuntimePort {
|
|||
tools: input.tools,
|
||||
});
|
||||
const startedAt = Date.now();
|
||||
const result = await collectResultWithRateLimitRetry({
|
||||
const result = await collectResult({
|
||||
query: this.runQuery,
|
||||
prompt: [input.system, input.prompt].filter(Boolean).join('\n\n'),
|
||||
options,
|
||||
allowedToolIds: new Set(mcpToolIds(input.tools ?? {})),
|
||||
expectedMcpServerNames: expectedMcpServerNames(input.tools),
|
||||
rateLimitGovernor: this.deps.rateLimitGovernor,
|
||||
abortSignal: input.abortSignal,
|
||||
});
|
||||
input.onMetrics?.({ totalMs: Date.now() - startedAt, usage: claudeTokenUsage(result) });
|
||||
const error = resultError(result);
|
||||
|
|
@ -413,14 +289,12 @@ export class ClaudeCodeKtxLlmRuntime implements KtxLlmRuntimePort {
|
|||
outputFormat: { type: 'json_schema' as const, schema: jsonSchema(input.schema as z.ZodType) },
|
||||
};
|
||||
const startedAt = Date.now();
|
||||
const result = await collectResultWithRateLimitRetry({
|
||||
const result = await collectResult({
|
||||
query: this.runQuery,
|
||||
prompt: [input.system, input.prompt].filter(Boolean).join('\n\n'),
|
||||
options,
|
||||
allowedToolIds: new Set([...mcpToolIds(input.tools ?? {}), STRUCTURED_OUTPUT_TOOL_NAME]),
|
||||
expectedMcpServerNames: expectedMcpServerNames(input.tools),
|
||||
rateLimitGovernor: this.deps.rateLimitGovernor,
|
||||
abortSignal: input.abortSignal,
|
||||
});
|
||||
input.onMetrics?.({ totalMs: Date.now() - startedAt, usage: claudeTokenUsage(result) });
|
||||
const error = resultError(result);
|
||||
|
|
@ -445,14 +319,12 @@ export class ClaudeCodeKtxLlmRuntime implements KtxLlmRuntimePort {
|
|||
maxTurns: params.stepBudget,
|
||||
tools: params.toolSet,
|
||||
});
|
||||
const result = await collectResultWithRateLimitRetry({
|
||||
const result = await collectResult({
|
||||
query: this.runQuery,
|
||||
prompt: params.userPrompt,
|
||||
options: { ...options, systemPrompt: params.systemPrompt },
|
||||
allowedToolIds: new Set(mcpToolIds(params.toolSet)),
|
||||
expectedMcpServerNames: expectedMcpServerNames(params.toolSet),
|
||||
rateLimitGovernor: this.deps.rateLimitGovernor,
|
||||
abortSignal: params.abortSignal,
|
||||
onAssistantTurn: async () => {
|
||||
stepIndex += 1;
|
||||
stepBoundariesMs.push(Date.now() - startedAt);
|
||||
|
|
@ -483,9 +355,6 @@ export class ClaudeCodeKtxLlmRuntime implements KtxLlmRuntimePort {
|
|||
},
|
||||
};
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) {
|
||||
throw error;
|
||||
}
|
||||
const err = error instanceof Error ? error : new Error(String(error));
|
||||
return {
|
||||
stopReason: 'error',
|
||||
|
|
@ -519,7 +388,7 @@ export async function runClaudeCodeAuthProbe(input: {
|
|||
env: input.env,
|
||||
maxTurns: 1,
|
||||
});
|
||||
const result = await collectResultWithRateLimitRetry({
|
||||
const result = await collectResult({
|
||||
query: input.query ?? defaultQuery,
|
||||
prompt: 'Reply with exactly: ok',
|
||||
options,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
import { z } from 'zod';
|
||||
import { noopLogger, type KtxLogger } from '../core/config.js';
|
||||
import { isAbortError, linkAbortSignal } from '../core/abort.js';
|
||||
import { isCompletedAgentStep, summarizeCodexExecEvents, type CodexExecEventSummary } from './codex-exec-events.js';
|
||||
import {
|
||||
startCodexRuntimeMcpServer,
|
||||
|
|
@ -9,7 +8,6 @@ import {
|
|||
import { resolveCodexModel } from './codex-models.js';
|
||||
import { buildCodexRuntimeConfig } from './codex-runtime-config.js';
|
||||
import { CodexSdkCliRunner, type CodexSdkRunner } from './codex-sdk-runner.js';
|
||||
import type { RateLimitGovernor } from './rate-limit-governor.js';
|
||||
import type {
|
||||
KtxGenerateObjectInput,
|
||||
KtxGenerateTextInput,
|
||||
|
|
@ -26,7 +24,6 @@ export interface CodexKtxLlmRuntimeDeps {
|
|||
runner?: CodexSdkRunner;
|
||||
startMcpServer?: (input: { projectDir: string; toolSet: KtxRuntimeToolSet }) => Promise<CodexRuntimeMcpServerHandle>;
|
||||
logger?: KtxLogger;
|
||||
rateLimitGovernor?: Pick<RateLimitGovernor, 'waitForReady' | 'report' | 'maxRetryAttempts'>;
|
||||
}
|
||||
|
||||
function modelForRole(modelSlots: CodexKtxLlmRuntimeDeps['modelSlots'], role: string): string {
|
||||
|
|
@ -162,12 +159,6 @@ function runtimeToolNames(toolSet: KtxRuntimeToolSet | undefined): string[] {
|
|||
return Object.values(toolSet ?? {}).map((descriptor) => descriptor.name);
|
||||
}
|
||||
|
||||
const CODEX_RATE_LIMIT_MARKERS = /\b429\b|rate limit|too many requests|quota exceeded|temporarily overloaded/i;
|
||||
|
||||
function isCodexRateLimitError(error: Error | undefined): boolean {
|
||||
return !!error && CODEX_RATE_LIMIT_MARKERS.test(error.message);
|
||||
}
|
||||
|
||||
export class CodexKtxLlmRuntime implements KtxLlmRuntimePort {
|
||||
private readonly runner: CodexSdkRunner;
|
||||
private readonly logger: KtxLogger;
|
||||
|
|
@ -177,37 +168,6 @@ export class CodexKtxLlmRuntime implements KtxLlmRuntimePort {
|
|||
this.logger = deps.logger ?? noopLogger;
|
||||
}
|
||||
|
||||
private async runWithRateLimitRetry<T>(
|
||||
abortSignal: AbortSignal | undefined,
|
||||
run: () => Promise<T>,
|
||||
getError: (result: T) => Error | undefined,
|
||||
): Promise<T> {
|
||||
// maxRetryAttempts() returns 1 when no governor is present or pacing is
|
||||
// disabled, so an opaque rate-limit failure surfaces on the first attempt
|
||||
// instead of being retried with no backoff.
|
||||
const maxAttempts = this.deps.rateLimitGovernor?.maxRetryAttempts() ?? 1;
|
||||
for (let attempt = 0; ; attempt += 1) {
|
||||
await this.deps.rateLimitGovernor?.waitForReady(abortSignal);
|
||||
const lastAttempt = attempt >= maxAttempts - 1;
|
||||
try {
|
||||
const result = await run();
|
||||
const error = getError(result);
|
||||
if (!isCodexRateLimitError(error) || lastAttempt) {
|
||||
return result;
|
||||
}
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) {
|
||||
throw error;
|
||||
}
|
||||
const err = error instanceof Error ? error : new Error(String(error));
|
||||
if (!isCodexRateLimitError(err) || lastAttempt) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
this.deps.rateLimitGovernor?.report({ provider: 'codex', status: 'rejected', rateLimitType: 'opaque' });
|
||||
}
|
||||
}
|
||||
|
||||
async generateText(input: KtxGenerateTextInput): Promise<string> {
|
||||
const startedAt = Date.now();
|
||||
const model = modelForRole(this.deps.modelSlots, input.role);
|
||||
|
|
@ -230,26 +190,18 @@ export class CodexKtxLlmRuntime implements KtxLlmRuntimePort {
|
|||
}
|
||||
: {}),
|
||||
});
|
||||
const result = await this.runWithRateLimitRetry(
|
||||
input.abortSignal,
|
||||
async () => {
|
||||
const collected = await collectEvents(
|
||||
await this.runner.runStreamed({
|
||||
projectDir: this.deps.projectDir,
|
||||
model,
|
||||
prompt: promptWithSystem(input.system, input.prompt),
|
||||
configOverrides: config.configOverrides,
|
||||
env: config.env,
|
||||
...(input.abortSignal ? { signal: input.abortSignal } : {}),
|
||||
}),
|
||||
);
|
||||
const summary = summarizeCodexExecEvents(collected.events, { startedAt });
|
||||
return { collected, summary };
|
||||
},
|
||||
({ collected, summary }) => summaryError(summary, collected.streamError),
|
||||
const collected = await collectEvents(
|
||||
await this.runner.runStreamed({
|
||||
projectDir: this.deps.projectDir,
|
||||
model,
|
||||
prompt: promptWithSystem(input.system, input.prompt),
|
||||
configOverrides: config.configOverrides,
|
||||
env: config.env,
|
||||
}),
|
||||
);
|
||||
input.onMetrics?.(metrics(result.summary, startedAt));
|
||||
return assertSuccessfulText(result.summary, result.collected.streamError);
|
||||
const summary = summarizeCodexExecEvents(collected.events, { startedAt });
|
||||
input.onMetrics?.(metrics(summary, startedAt));
|
||||
return assertSuccessfulText(summary, collected.streamError);
|
||||
} finally {
|
||||
await mcp?.close();
|
||||
}
|
||||
|
|
@ -279,27 +231,19 @@ export class CodexKtxLlmRuntime implements KtxLlmRuntimePort {
|
|||
}
|
||||
: {}),
|
||||
});
|
||||
const result = await this.runWithRateLimitRetry(
|
||||
input.abortSignal,
|
||||
async () => {
|
||||
const collected = await collectEvents(
|
||||
await this.runner.runStreamed({
|
||||
projectDir: this.deps.projectDir,
|
||||
model,
|
||||
prompt: promptWithSystem(input.system, input.prompt),
|
||||
configOverrides: config.configOverrides,
|
||||
env: config.env,
|
||||
outputSchema: z.toJSONSchema(input.schema, { target: 'draft-7' }) as Record<string, unknown>,
|
||||
...(input.abortSignal ? { signal: input.abortSignal } : {}),
|
||||
}),
|
||||
);
|
||||
const summary = summarizeCodexExecEvents(collected.events, { startedAt });
|
||||
return { collected, summary };
|
||||
},
|
||||
({ collected, summary }) => summaryError(summary, collected.streamError),
|
||||
const collected = await collectEvents(
|
||||
await this.runner.runStreamed({
|
||||
projectDir: this.deps.projectDir,
|
||||
model,
|
||||
prompt: promptWithSystem(input.system, input.prompt),
|
||||
configOverrides: config.configOverrides,
|
||||
env: config.env,
|
||||
outputSchema: z.toJSONSchema(input.schema, { target: 'draft-7' }) as Record<string, unknown>,
|
||||
}),
|
||||
);
|
||||
input.onMetrics?.(metrics(result.summary, startedAt));
|
||||
return parseStructuredOutput(input.schema, assertSuccessfulText(result.summary, result.collected.streamError));
|
||||
const summary = summarizeCodexExecEvents(collected.events, { startedAt });
|
||||
input.onMetrics?.(metrics(summary, startedAt));
|
||||
return parseStructuredOutput(input.schema, assertSuccessfulText(summary, collected.streamError));
|
||||
} finally {
|
||||
await mcp?.close();
|
||||
}
|
||||
|
|
@ -328,6 +272,7 @@ export class CodexKtxLlmRuntime implements KtxLlmRuntimePort {
|
|||
}
|
||||
: {}),
|
||||
});
|
||||
const abortController = new AbortController();
|
||||
const onStep = async (stepIndex: number): Promise<void> => {
|
||||
try {
|
||||
await params.onStepFinish?.({ stepIndex, stepBudget: params.stepBudget });
|
||||
|
|
@ -337,50 +282,31 @@ export class CodexKtxLlmRuntime implements KtxLlmRuntimePort {
|
|||
);
|
||||
}
|
||||
};
|
||||
const result = await this.runWithRateLimitRetry(
|
||||
params.abortSignal,
|
||||
async () => {
|
||||
const linked = linkAbortSignal(params.abortSignal);
|
||||
const abortController = linked.controller;
|
||||
try {
|
||||
const collected = await collectEvents(
|
||||
await this.runner.runStreamed({
|
||||
projectDir: this.deps.projectDir,
|
||||
model,
|
||||
prompt: promptWithSystem(params.systemPrompt, params.userPrompt),
|
||||
configOverrides: config.configOverrides,
|
||||
env: config.env,
|
||||
signal: abortController.signal,
|
||||
}),
|
||||
{ stepBudget: params.stepBudget, abortController, onStep },
|
||||
);
|
||||
const summary = summarizeCodexExecEvents(collected.events, { startedAt });
|
||||
return { collected, summary };
|
||||
} finally {
|
||||
linked.dispose();
|
||||
}
|
||||
},
|
||||
({ collected, summary }) => summaryError(summary, collected.streamError),
|
||||
const collected = await collectEvents(
|
||||
await this.runner.runStreamed({
|
||||
projectDir: this.deps.projectDir,
|
||||
model,
|
||||
prompt: promptWithSystem(params.systemPrompt, params.userPrompt),
|
||||
configOverrides: config.configOverrides,
|
||||
env: config.env,
|
||||
signal: abortController.signal,
|
||||
}),
|
||||
{ stepBudget: params.stepBudget, abortController, onStep },
|
||||
);
|
||||
const error = summaryError(result.summary, result.collected.streamError);
|
||||
if (isAbortError(error)) {
|
||||
throw error;
|
||||
}
|
||||
const stopReason = result.collected.budgetExceeded ? 'budget' : error ? 'error' : result.summary.stopReason;
|
||||
const summary = summarizeCodexExecEvents(collected.events, { startedAt });
|
||||
const error = summaryError(summary, collected.streamError);
|
||||
const stopReason = collected.budgetExceeded ? 'budget' : error ? 'error' : summary.stopReason;
|
||||
return {
|
||||
stopReason,
|
||||
...(stopReason === 'error' && error ? { error } : {}),
|
||||
metrics: {
|
||||
totalMs: Date.now() - startedAt,
|
||||
usage: result.summary.usage,
|
||||
stepCount: result.summary.stepCount,
|
||||
stepBoundariesMs: result.summary.stepBoundariesMs,
|
||||
usage: summary.usage,
|
||||
stepCount: summary.stepCount,
|
||||
stepBoundariesMs: summary.stepBoundariesMs,
|
||||
},
|
||||
};
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) {
|
||||
throw error;
|
||||
}
|
||||
const err = error instanceof Error ? error : new Error(String(error));
|
||||
return {
|
||||
stopReason: 'error',
|
||||
|
|
|
|||
|
|
@ -6,28 +6,16 @@ import type { KtxProjectEmbeddingConfig, KtxProjectLlmConfig } from '../project/
|
|||
import { AiSdkKtxLlmRuntime } from './ai-sdk-runtime.js';
|
||||
import { ClaudeCodeKtxLlmRuntime } from './claude-code-runtime.js';
|
||||
import { CodexKtxLlmRuntime } from './codex-runtime.js';
|
||||
import type { RateLimitGovernor } from './rate-limit-governor.js';
|
||||
import type { KtxLlmRuntimePort } from './runtime-port.js';
|
||||
|
||||
type ClaudeCodeRuntimeDeps = ConstructorParameters<typeof ClaudeCodeKtxLlmRuntime>[0] & {
|
||||
rateLimitGovernor?: RateLimitGovernor;
|
||||
};
|
||||
type CodexRuntimeDeps = ConstructorParameters<typeof CodexKtxLlmRuntime>[0] & {
|
||||
rateLimitGovernor?: RateLimitGovernor;
|
||||
};
|
||||
type AiSdkRuntimeDeps = ConstructorParameters<typeof AiSdkKtxLlmRuntime>[0] & {
|
||||
rateLimitGovernor?: RateLimitGovernor;
|
||||
};
|
||||
|
||||
interface LocalConfigDeps {
|
||||
env?: NodeJS.ProcessEnv;
|
||||
projectDir?: string;
|
||||
rateLimitGovernor?: RateLimitGovernor;
|
||||
createKtxLlmProvider?: typeof createKtxLlmProvider;
|
||||
createKtxEmbeddingProvider?: typeof createKtxEmbeddingProvider;
|
||||
createClaudeCodeRuntime?: (deps: ClaudeCodeRuntimeDeps) => KtxLlmRuntimePort;
|
||||
createCodexRuntime?: (deps: CodexRuntimeDeps) => KtxLlmRuntimePort;
|
||||
createAiSdkRuntime?: (deps: AiSdkRuntimeDeps) => KtxLlmRuntimePort;
|
||||
createClaudeCodeRuntime?: (deps: ConstructorParameters<typeof ClaudeCodeKtxLlmRuntime>[0]) => KtxLlmRuntimePort;
|
||||
createCodexRuntime?: (deps: ConstructorParameters<typeof CodexKtxLlmRuntime>[0]) => KtxLlmRuntimePort;
|
||||
createAiSdkRuntime?: (deps: { llmProvider: KtxLlmProvider }) => KtxLlmRuntimePort;
|
||||
}
|
||||
|
||||
function resolveOptional(value: string | undefined, env: NodeJS.ProcessEnv): string | undefined {
|
||||
|
|
@ -141,7 +129,6 @@ export function createLocalKtxLlmRuntimeFromConfig(
|
|||
projectDir,
|
||||
modelSlots: resolved.modelSlots,
|
||||
env: deps.env,
|
||||
rateLimitGovernor: deps.rateLimitGovernor,
|
||||
});
|
||||
}
|
||||
if (resolved.backend === 'codex') {
|
||||
|
|
@ -152,14 +139,10 @@ export function createLocalKtxLlmRuntimeFromConfig(
|
|||
return (deps.createCodexRuntime ?? ((runtimeDeps) => new CodexKtxLlmRuntime(runtimeDeps)))({
|
||||
projectDir,
|
||||
modelSlots: resolved.modelSlots,
|
||||
rateLimitGovernor: deps.rateLimitGovernor,
|
||||
});
|
||||
}
|
||||
const llmProvider = (deps.createKtxLlmProvider ?? createKtxLlmProvider)(resolved);
|
||||
return (deps.createAiSdkRuntime ?? ((runtimeDeps) => new AiSdkKtxLlmRuntime(runtimeDeps)))({
|
||||
llmProvider,
|
||||
rateLimitGovernor: deps.rateLimitGovernor,
|
||||
});
|
||||
return (deps.createAiSdkRuntime ?? ((runtimeDeps) => new AiSdkKtxLlmRuntime(runtimeDeps)))({ llmProvider });
|
||||
}
|
||||
|
||||
export function resolveLocalKtxEmbeddingConfig(
|
||||
|
|
|
|||
|
|
@ -1,387 +0,0 @@
|
|||
import { createAbortError, throwIfAborted } from '../core/abort.js';
|
||||
|
||||
export type RateLimitProvider = 'claude-subscription' | 'anthropic-api' | 'vertex' | 'codex';
|
||||
type RateLimitSignalStatus = 'allowed' | 'warning' | 'rejected';
|
||||
|
||||
export interface RateLimitSignal {
|
||||
provider: RateLimitProvider;
|
||||
status: RateLimitSignalStatus;
|
||||
resetAtMs?: number;
|
||||
retryAfterMs?: number;
|
||||
utilization?: number;
|
||||
rateLimitType?: string;
|
||||
}
|
||||
|
||||
export interface RateLimitRetryConfig {
|
||||
maxAttempts: number;
|
||||
baseDelayMs: number;
|
||||
maxDelayMs: number;
|
||||
jitter: boolean;
|
||||
}
|
||||
|
||||
export interface RateLimitGovernorConfig {
|
||||
enabled: boolean;
|
||||
maxConcurrency: number;
|
||||
throttleThreshold: number;
|
||||
minConcurrencyUnderPressure: number;
|
||||
maxWaitMs?: number;
|
||||
waitStateTickMs: number;
|
||||
retry: RateLimitRetryConfig;
|
||||
}
|
||||
|
||||
export type RateLimitWaitState =
|
||||
| {
|
||||
kind: 'rate_limit_observed';
|
||||
provider: RateLimitProvider;
|
||||
status: RateLimitSignalStatus;
|
||||
rateLimitType?: string;
|
||||
resetAtMs?: number;
|
||||
retryAfterMs?: number;
|
||||
utilization?: number;
|
||||
}
|
||||
| {
|
||||
kind: 'concurrency_adjusted';
|
||||
provider: RateLimitProvider;
|
||||
from: number;
|
||||
to: number;
|
||||
reason: string;
|
||||
rateLimitType?: string;
|
||||
utilization?: number;
|
||||
}
|
||||
| {
|
||||
kind: 'wait_started' | 'wait_tick' | 'wait_finished';
|
||||
provider: RateLimitProvider;
|
||||
rateLimitType?: string;
|
||||
resumeAtMs: number;
|
||||
remainingMs: number;
|
||||
};
|
||||
|
||||
export interface RateLimitGovernorDeps {
|
||||
now?: () => number;
|
||||
sleep?: (ms: number, signal?: AbortSignal) => Promise<void>;
|
||||
random?: () => number;
|
||||
}
|
||||
|
||||
export type RateLimitRelease = () => void;
|
||||
type Subscriber = (state: RateLimitWaitState) => void;
|
||||
|
||||
const defaultSleep = (ms: number, signal?: AbortSignal): Promise<void> =>
|
||||
new Promise((resolve, reject) => {
|
||||
if (signal?.aborted) {
|
||||
reject(createAbortError());
|
||||
return;
|
||||
}
|
||||
const timeout = setTimeout(resolve, ms);
|
||||
signal?.addEventListener(
|
||||
'abort',
|
||||
() => {
|
||||
clearTimeout(timeout);
|
||||
reject(createAbortError());
|
||||
},
|
||||
{ once: true },
|
||||
);
|
||||
});
|
||||
|
||||
export function createRateLimitGovernorConfig(
|
||||
input: Partial<RateLimitGovernorConfig> & { retry?: Partial<RateLimitRetryConfig> } = {},
|
||||
): RateLimitGovernorConfig {
|
||||
return {
|
||||
enabled: input.enabled ?? true,
|
||||
maxConcurrency: input.maxConcurrency ?? 1,
|
||||
throttleThreshold: input.throttleThreshold ?? 0.8,
|
||||
minConcurrencyUnderPressure: input.minConcurrencyUnderPressure ?? 1,
|
||||
...(input.maxWaitMs !== undefined ? { maxWaitMs: input.maxWaitMs } : {}),
|
||||
waitStateTickMs: input.waitStateTickMs ?? 1_000,
|
||||
retry: {
|
||||
maxAttempts: input.retry?.maxAttempts ?? 6,
|
||||
baseDelayMs: input.retry?.baseDelayMs ?? 1_000,
|
||||
maxDelayMs: input.retry?.maxDelayMs ?? 60_000,
|
||||
jitter: input.retry?.jitter ?? true,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export class RateLimitGovernor {
|
||||
private readonly now: () => number;
|
||||
private readonly sleep: (ms: number, signal?: AbortSignal) => Promise<void>;
|
||||
private readonly random: () => number;
|
||||
private readonly subscribers = new Set<Subscriber>();
|
||||
private waiters: Array<() => void> = [];
|
||||
private active = 0;
|
||||
private effectiveLimit: number;
|
||||
private pausedUntilMs: number | null = null;
|
||||
private pausedProvider: RateLimitProvider | null = null;
|
||||
private pausedRateLimitType: string | undefined;
|
||||
private pausedTickMs: number | null = null;
|
||||
private opaqueAttempts = new Map<RateLimitProvider, number>();
|
||||
private pauseGeneration = 0;
|
||||
private visibleWaitAbort: AbortController | null = null;
|
||||
|
||||
constructor(
|
||||
private readonly config: RateLimitGovernorConfig,
|
||||
deps: RateLimitGovernorDeps = {},
|
||||
) {
|
||||
this.now = deps.now ?? Date.now;
|
||||
this.sleep = deps.sleep ?? defaultSleep;
|
||||
this.random = deps.random ?? Math.random;
|
||||
this.effectiveLimit = Math.max(1, config.maxConcurrency);
|
||||
}
|
||||
|
||||
currentLimit(): number {
|
||||
return this.config.enabled ? this.effectiveLimit : this.config.maxConcurrency;
|
||||
}
|
||||
|
||||
/**
|
||||
* Total attempts a runtime should make for a single rate-limited LLM call,
|
||||
* including the first try. Returns 1 (no outer retry) when pacing is disabled:
|
||||
* the outer retry loop only exists to cooperate with this governor's pause, so
|
||||
* without active pacing there is no backoff to apply and the backend's own
|
||||
* retry handles transient rejections.
|
||||
*/
|
||||
maxRetryAttempts(): number {
|
||||
return this.config.enabled ? Math.max(1, this.config.retry.maxAttempts) : 1;
|
||||
}
|
||||
|
||||
activeSlots(): number {
|
||||
return this.active;
|
||||
}
|
||||
|
||||
subscribe(cb: Subscriber): () => void {
|
||||
this.subscribers.add(cb);
|
||||
if (this.pausedUntilMs !== null) {
|
||||
this.startVisibleWaitTicker();
|
||||
}
|
||||
return () => {
|
||||
this.subscribers.delete(cb);
|
||||
if (this.subscribers.size === 0) {
|
||||
this.stopVisibleWaitTicker();
|
||||
this.wakeWaiters();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
report(signal: RateLimitSignal): void {
|
||||
if (!this.config.enabled) {
|
||||
return;
|
||||
}
|
||||
this.emit({
|
||||
kind: 'rate_limit_observed',
|
||||
provider: signal.provider,
|
||||
status: signal.status,
|
||||
...(signal.rateLimitType ? { rateLimitType: signal.rateLimitType } : {}),
|
||||
...(signal.resetAtMs !== undefined ? { resetAtMs: signal.resetAtMs } : {}),
|
||||
...(signal.retryAfterMs !== undefined ? { retryAfterMs: signal.retryAfterMs } : {}),
|
||||
...(signal.utilization !== undefined ? { utilization: signal.utilization } : {}),
|
||||
});
|
||||
|
||||
if (signal.status === 'rejected') {
|
||||
this.applyPause(signal);
|
||||
return;
|
||||
}
|
||||
|
||||
if (signal.status === 'warning' || (signal.utilization ?? 0) >= this.config.throttleThreshold) {
|
||||
this.adjustLimit(Math.max(1, this.config.minConcurrencyUnderPressure), signal, 'provider pressure');
|
||||
return;
|
||||
}
|
||||
|
||||
this.opaqueAttempts.delete(signal.provider);
|
||||
if ((signal.utilization ?? 0) < this.config.throttleThreshold) {
|
||||
this.adjustLimit(Math.max(1, this.config.maxConcurrency), signal, 'provider recovered');
|
||||
}
|
||||
}
|
||||
|
||||
async waitForReady(signal?: AbortSignal): Promise<void> {
|
||||
throwIfAborted(signal);
|
||||
if (!this.config.enabled) {
|
||||
return;
|
||||
}
|
||||
await this.waitForPause(signal);
|
||||
throwIfAborted(signal);
|
||||
}
|
||||
|
||||
async acquireWorkSlot(signal?: AbortSignal): Promise<RateLimitRelease> {
|
||||
throwIfAborted(signal);
|
||||
if (!this.config.enabled) {
|
||||
this.active += 1;
|
||||
return () => {
|
||||
this.active -= 1;
|
||||
};
|
||||
}
|
||||
|
||||
while (true) {
|
||||
throwIfAborted(signal);
|
||||
await this.waitForPause(signal);
|
||||
throwIfAborted(signal);
|
||||
if (this.active < this.effectiveLimit) {
|
||||
this.active += 1;
|
||||
let released = false;
|
||||
return () => {
|
||||
if (released) return;
|
||||
released = true;
|
||||
this.active -= 1;
|
||||
this.wakeWaiters();
|
||||
};
|
||||
}
|
||||
await this.waitForSlot(signal);
|
||||
}
|
||||
}
|
||||
|
||||
private applyPause(signal: RateLimitSignal): void {
|
||||
const resumeAtMs = this.resumeAtMsFor(signal);
|
||||
const boundedResumeAtMs =
|
||||
this.config.maxWaitMs === undefined ? resumeAtMs : Math.min(resumeAtMs, this.now() + this.config.maxWaitMs);
|
||||
if (this.pausedUntilMs === null || boundedResumeAtMs > this.pausedUntilMs) {
|
||||
this.pausedUntilMs = boundedResumeAtMs;
|
||||
this.pausedProvider = signal.provider;
|
||||
this.pausedRateLimitType = signal.rateLimitType;
|
||||
this.pausedTickMs = signal.rateLimitType === 'opaque' ? Math.max(1, boundedResumeAtMs - this.now()) : null;
|
||||
this.emitWait('wait_started');
|
||||
this.startVisibleWaitTicker();
|
||||
this.wakeWaiters();
|
||||
}
|
||||
this.adjustLimit(Math.max(1, this.config.minConcurrencyUnderPressure), signal, 'provider rejected');
|
||||
}
|
||||
|
||||
private resumeAtMsFor(signal: RateLimitSignal): number {
|
||||
if (signal.resetAtMs !== undefined) {
|
||||
return signal.resetAtMs;
|
||||
}
|
||||
if (signal.retryAfterMs !== undefined) {
|
||||
return this.now() + signal.retryAfterMs;
|
||||
}
|
||||
const attempts = this.opaqueAttempts.get(signal.provider) ?? 0;
|
||||
this.opaqueAttempts.set(signal.provider, Math.min(attempts + 1, this.config.retry.maxAttempts));
|
||||
const base = Math.min(
|
||||
this.config.retry.maxDelayMs,
|
||||
this.config.retry.baseDelayMs * 2 ** Math.min(attempts, this.config.retry.maxAttempts - 1),
|
||||
);
|
||||
const jitterMultiplier = this.config.retry.jitter ? 0.75 + this.random() * 0.5 : 1;
|
||||
return this.now() + Math.round(base * jitterMultiplier);
|
||||
}
|
||||
|
||||
private adjustLimit(to: number, signal: RateLimitSignal, reason: string): void {
|
||||
const bounded = Math.max(1, Math.min(this.config.maxConcurrency, to));
|
||||
if (bounded === this.effectiveLimit) {
|
||||
return;
|
||||
}
|
||||
const from = this.effectiveLimit;
|
||||
this.effectiveLimit = bounded;
|
||||
this.emit({
|
||||
kind: 'concurrency_adjusted',
|
||||
provider: signal.provider,
|
||||
from,
|
||||
to: bounded,
|
||||
reason,
|
||||
...(signal.rateLimitType ? { rateLimitType: signal.rateLimitType } : {}),
|
||||
...(signal.utilization !== undefined ? { utilization: signal.utilization } : {}),
|
||||
});
|
||||
this.wakeWaiters();
|
||||
}
|
||||
|
||||
private startVisibleWaitTicker(): void {
|
||||
if (this.subscribers.size === 0 || this.pausedUntilMs === null) {
|
||||
return;
|
||||
}
|
||||
this.stopVisibleWaitTicker();
|
||||
const generation = (this.pauseGeneration += 1);
|
||||
const controller = new AbortController();
|
||||
this.visibleWaitAbort = controller;
|
||||
void this.runVisibleWaitTicker(generation, controller.signal).catch(() => undefined);
|
||||
}
|
||||
|
||||
private stopVisibleWaitTicker(): void {
|
||||
this.visibleWaitAbort?.abort();
|
||||
this.visibleWaitAbort = null;
|
||||
}
|
||||
|
||||
private async runVisibleWaitTicker(generation: number, signal: AbortSignal): Promise<void> {
|
||||
while (!signal.aborted && generation === this.pauseGeneration && this.pausedUntilMs !== null) {
|
||||
const remainingMs = this.pausedUntilMs - this.now();
|
||||
if (remainingMs <= 0) {
|
||||
this.finishPause(generation);
|
||||
return;
|
||||
}
|
||||
this.emitWait('wait_tick');
|
||||
await this.sleep(Math.min(this.pausedTickMs ?? this.config.waitStateTickMs, remainingMs), signal);
|
||||
}
|
||||
}
|
||||
|
||||
private finishPause(generation?: number): void {
|
||||
if (generation !== undefined && generation !== this.pauseGeneration) {
|
||||
return;
|
||||
}
|
||||
this.emitWait('wait_finished');
|
||||
this.pausedUntilMs = null;
|
||||
this.pausedProvider = null;
|
||||
this.pausedRateLimitType = undefined;
|
||||
this.pausedTickMs = null;
|
||||
this.stopVisibleWaitTicker();
|
||||
this.wakeWaiters();
|
||||
}
|
||||
|
||||
private async waitForPause(signal?: AbortSignal): Promise<void> {
|
||||
throwIfAborted(signal);
|
||||
while (this.pausedUntilMs !== null) {
|
||||
const remainingMs = this.pausedUntilMs - this.now();
|
||||
if (remainingMs <= 0) {
|
||||
this.finishPause();
|
||||
return;
|
||||
}
|
||||
if (this.visibleWaitAbort !== null) {
|
||||
await this.waitForSlot(signal);
|
||||
} else {
|
||||
await this.sleep(Math.min(this.pausedTickMs ?? this.config.waitStateTickMs, remainingMs), signal);
|
||||
}
|
||||
throwIfAborted(signal);
|
||||
}
|
||||
}
|
||||
|
||||
private waitForSlot(signal?: AbortSignal): Promise<void> {
|
||||
if (signal?.aborted) {
|
||||
return Promise.reject(createAbortError());
|
||||
}
|
||||
return new Promise((resolve, reject) => {
|
||||
const wake = () => {
|
||||
cleanup();
|
||||
resolve();
|
||||
};
|
||||
const onAbort = () => {
|
||||
cleanup();
|
||||
reject(createAbortError());
|
||||
};
|
||||
const cleanup = () => {
|
||||
this.waiters = this.waiters.filter((candidate) => candidate !== wake);
|
||||
signal?.removeEventListener('abort', onAbort);
|
||||
};
|
||||
this.waiters.push(wake);
|
||||
signal?.addEventListener('abort', onAbort, { once: true });
|
||||
});
|
||||
}
|
||||
|
||||
private wakeWaiters(): void {
|
||||
const waiters = this.waiters;
|
||||
this.waiters = [];
|
||||
for (const waiter of waiters) {
|
||||
waiter();
|
||||
}
|
||||
}
|
||||
|
||||
private emitWait(kind: Extract<RateLimitWaitState['kind'], 'wait_started' | 'wait_tick' | 'wait_finished'>): void {
|
||||
if (this.pausedUntilMs === null || this.pausedProvider === null) {
|
||||
return;
|
||||
}
|
||||
this.emit({
|
||||
kind,
|
||||
provider: this.pausedProvider,
|
||||
...(this.pausedRateLimitType ? { rateLimitType: this.pausedRateLimitType } : {}),
|
||||
resumeAtMs: this.pausedUntilMs,
|
||||
remainingMs: Math.max(0, this.pausedUntilMs - this.now()),
|
||||
});
|
||||
}
|
||||
|
||||
private emit(state: RateLimitWaitState): void {
|
||||
for (const subscriber of this.subscribers) {
|
||||
subscriber(state);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -49,7 +49,6 @@ export interface RunLoopParams {
|
|||
stepBudget: number;
|
||||
telemetryTags: Record<string, string>;
|
||||
onStepFinish?: (info: RunLoopStepInfo) => void | Promise<void>;
|
||||
abortSignal?: AbortSignal;
|
||||
}
|
||||
|
||||
export interface RunLoopResult {
|
||||
|
|
@ -65,7 +64,6 @@ export interface KtxGenerateTextInput {
|
|||
tools?: KtxRuntimeToolSet;
|
||||
temperature?: number;
|
||||
onMetrics?: (metrics: { totalMs: number; usage: LlmTokenUsage }) => void;
|
||||
abortSignal?: AbortSignal;
|
||||
}
|
||||
|
||||
export interface KtxGenerateObjectInput<TOutput, TSchema extends z.ZodType<TOutput>> {
|
||||
|
|
@ -76,7 +74,6 @@ export interface KtxGenerateObjectInput<TOutput, TSchema extends z.ZodType<TOutp
|
|||
temperature?: number;
|
||||
schema: TSchema;
|
||||
onMetrics?: (metrics: { totalMs: number; usage: LlmTokenUsage }) => void;
|
||||
abortSignal?: AbortSignal;
|
||||
}
|
||||
|
||||
export interface KtxLlmRuntimePort {
|
||||
|
|
|
|||
|
|
@ -3,13 +3,7 @@ import type { ToolAnnotations } from '@modelcontextprotocol/sdk/types.js';
|
|||
import { z } from 'zod';
|
||||
import type { KtxCliIo } from '../../cli-runtime.js';
|
||||
import type { MemoryAgentInput } from '../../context/memory/types.js';
|
||||
import {
|
||||
emitTelemetryEvent,
|
||||
mcpTelemetrySampleRate,
|
||||
reportException,
|
||||
shouldEmitMcpTelemetry,
|
||||
} from '../../telemetry/index.js';
|
||||
import { collectTelemetryRedactionSecrets } from '../../telemetry/redaction-secrets.js';
|
||||
import { emitTelemetryEvent, mcpTelemetrySampleRate, shouldEmitMcpTelemetry } from '../../telemetry/index.js';
|
||||
import { scrubErrorClass } from '../../telemetry/scrubber.js';
|
||||
import type {
|
||||
KtxMcpClientInfo,
|
||||
|
|
@ -524,26 +518,11 @@ function registerParsedTool<TSchema extends z.ZodType>(
|
|||
},
|
||||
schema: TSchema,
|
||||
handler: (input: z.infer<TSchema>, context?: KtxMcpToolHandlerContext) => Promise<KtxMcpToolResult>,
|
||||
telemetry?: { projectDir?: string; io?: KtxCliIo },
|
||||
): void {
|
||||
server.registerTool(name, config, async (input, context) => {
|
||||
try {
|
||||
return await handler(schema.parse(input), context);
|
||||
} catch (error) {
|
||||
if (telemetry?.io) {
|
||||
await reportException({
|
||||
error,
|
||||
context: { source: `mcp:${name}`, handled: true, fatal: false },
|
||||
projectDir: telemetry.projectDir,
|
||||
io: telemetry.io,
|
||||
redactionSecrets: await collectTelemetryRedactionSecrets({
|
||||
projectDir: telemetry.projectDir,
|
||||
includeLlm: true,
|
||||
includeEmbeddings: true,
|
||||
env: process.env,
|
||||
}),
|
||||
});
|
||||
}
|
||||
return jsonErrorToolResult(formatToolError(error));
|
||||
}
|
||||
});
|
||||
|
|
@ -592,20 +571,6 @@ function instrumentMcpServer(
|
|||
}
|
||||
return result;
|
||||
} catch (error) {
|
||||
if (telemetry.io) {
|
||||
await reportException({
|
||||
error,
|
||||
context: { source: `mcp:${name}`, handled: true, fatal: false },
|
||||
projectDir: telemetry.projectDir,
|
||||
io: telemetry.io,
|
||||
redactionSecrets: await collectTelemetryRedactionSecrets({
|
||||
projectDir: telemetry.projectDir,
|
||||
includeLlm: true,
|
||||
includeEmbeddings: true,
|
||||
env: process.env,
|
||||
}),
|
||||
});
|
||||
}
|
||||
if (telemetry.io && telemetry.projectDir && shouldEmitMcpTelemetry()) {
|
||||
const errorClass = scrubErrorClass(error);
|
||||
await emitTelemetryEvent({
|
||||
|
|
@ -631,7 +596,6 @@ function instrumentMcpServer(
|
|||
|
||||
export function registerKtxContextTools(deps: RegisterKtxContextToolsDeps): void {
|
||||
const { ports, userContext } = deps;
|
||||
const toolTelemetry = { projectDir: deps.projectDir, io: deps.io };
|
||||
const server = instrumentMcpServer(deps.server, {
|
||||
projectDir: deps.projectDir,
|
||||
io: deps.io,
|
||||
|
|
@ -652,7 +616,6 @@ export function registerKtxContextTools(deps: RegisterKtxContextToolsDeps): void
|
|||
},
|
||||
connectionListSchema,
|
||||
async () => jsonToolResult({ connections: await connections.list() }),
|
||||
toolTelemetry,
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -677,7 +640,6 @@ export function registerKtxContextTools(deps: RegisterKtxContextToolsDeps): void
|
|||
limit: input.limit,
|
||||
}),
|
||||
),
|
||||
toolTelemetry,
|
||||
);
|
||||
|
||||
registerParsedTool(
|
||||
|
|
@ -695,7 +657,6 @@ export function registerKtxContextTools(deps: RegisterKtxContextToolsDeps): void
|
|||
const page = await knowledge.read({ userId: userContext.userId, key: input.key });
|
||||
return page ? jsonToolResult(page) : jsonErrorToolResult(`Wiki page "${input.key}" was not found.`);
|
||||
},
|
||||
toolTelemetry,
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -718,7 +679,6 @@ export function registerKtxContextTools(deps: RegisterKtxContextToolsDeps): void
|
|||
? jsonToolResult(source)
|
||||
: jsonErrorToolResult(`Semantic-layer source "${input.sourceName}" was not found.`);
|
||||
},
|
||||
toolTelemetry,
|
||||
);
|
||||
|
||||
registerParsedTool(
|
||||
|
|
@ -751,7 +711,6 @@ export function registerKtxContextTools(deps: RegisterKtxContextToolsDeps): void
|
|||
);
|
||||
return jsonToolResult(projectSlQueryResult(result, input.include));
|
||||
},
|
||||
toolTelemetry,
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -769,7 +728,6 @@ export function registerKtxContextTools(deps: RegisterKtxContextToolsDeps): void
|
|||
},
|
||||
entityDetailsSchema,
|
||||
async (input) => jsonToolResult(await entityDetails.read(input)),
|
||||
toolTelemetry,
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -787,7 +745,6 @@ export function registerKtxContextTools(deps: RegisterKtxContextToolsDeps): void
|
|||
},
|
||||
dictionarySearchSchema,
|
||||
async (input) => jsonToolResult(await dictionarySearch.search(input)),
|
||||
toolTelemetry,
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -805,7 +762,6 @@ export function registerKtxContextTools(deps: RegisterKtxContextToolsDeps): void
|
|||
},
|
||||
discoverDataSchema,
|
||||
async (input) => jsonToolResult({ refs: await discover.search(input) }),
|
||||
toolTelemetry,
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -835,7 +791,6 @@ export function registerKtxContextTools(deps: RegisterKtxContextToolsDeps): void
|
|||
),
|
||||
);
|
||||
},
|
||||
toolTelemetry,
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -863,7 +818,6 @@ export function registerKtxContextTools(deps: RegisterKtxContextToolsDeps): void
|
|||
};
|
||||
return jsonToolResult(await memoryIngest.ingest(ingestInput));
|
||||
},
|
||||
toolTelemetry,
|
||||
);
|
||||
|
||||
registerParsedTool(
|
||||
|
|
@ -881,7 +835,6 @@ export function registerKtxContextTools(deps: RegisterKtxContextToolsDeps): void
|
|||
const status = await memoryIngest.status(input.runId);
|
||||
return status ? jsonToolResult(status) : jsonErrorToolResult(`Memory ingest run "${input.runId}" was not found.`);
|
||||
},
|
||||
toolTelemetry,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -100,44 +100,6 @@ const workUnitsSchema = z
|
|||
})
|
||||
.describe('Concurrency and failure handling for ingest work units.');
|
||||
|
||||
const ingestRateLimitRetrySchema = z
|
||||
.strictObject({
|
||||
maxAttempts: z
|
||||
.int()
|
||||
.positive()
|
||||
.default(6)
|
||||
.describe(
|
||||
'Maximum attempts for a single rate-limited LLM call before the failure surfaces, counting the first try. Also bounds how far opaque backoff grows for providers that do not expose a reset time.',
|
||||
),
|
||||
baseDelayMs: z.int().positive().default(1_000).describe('Initial opaque retry delay in milliseconds.'),
|
||||
maxDelayMs: z.int().positive().default(60_000).describe('Maximum opaque retry delay in milliseconds.'),
|
||||
jitter: z.boolean().default(true).describe('When true, apply bounded jitter to opaque retry delays.'),
|
||||
})
|
||||
.describe('Retry policy for rate-limit responses that do not include a reset time or retry-after value.');
|
||||
|
||||
const ingestRateLimitSchema = z
|
||||
.strictObject({
|
||||
enabled: z.boolean().default(true).describe('Master switch for ingest LLM rate-limit pacing and visible waits.'),
|
||||
throttleThreshold: z
|
||||
.number()
|
||||
.min(0)
|
||||
.max(1)
|
||||
.default(0.8)
|
||||
.describe('Provider utilization at or above which ingest throttles new work-unit starts.'),
|
||||
minConcurrencyUnderPressure: z
|
||||
.int()
|
||||
.positive()
|
||||
.default(1)
|
||||
.describe('Effective work-unit concurrency while a provider is under rate-limit pressure.'),
|
||||
maxWaitMs: z
|
||||
.int()
|
||||
.positive()
|
||||
.optional()
|
||||
.describe('Optional cap on a single provider reset wait. Omit to wait indefinitely until the provider reset time.'),
|
||||
retry: ingestRateLimitRetrySchema.prefault({}).describe('Opaque retry policy for providers without reset hints.'),
|
||||
})
|
||||
.describe('Rate-limit pacing and wait policy for ingest LLM calls.');
|
||||
|
||||
const ingestSchema = z
|
||||
.strictObject({
|
||||
adapters: z
|
||||
|
|
@ -148,7 +110,6 @@ const ingestSchema = z
|
|||
.prefault({ backend: 'none' })
|
||||
.describe('Embedding configuration used when ingest adapters need to embed documents.'),
|
||||
workUnits: workUnitsSchema.prefault({}).describe('Concurrency and failure handling for ingest work units.'),
|
||||
rateLimit: ingestRateLimitSchema.prefault({}).describe('LLM rate-limit pacing and visible-wait policy for ingest.'),
|
||||
profile: z
|
||||
.union([z.boolean(), z.literal('json')])
|
||||
.default(false)
|
||||
|
|
|
|||
|
|
@ -303,29 +303,9 @@ export interface KtxTableListEntry {
|
|||
kind: 'table' | 'view';
|
||||
}
|
||||
|
||||
export interface KtxConnectorTestResult {
|
||||
interface KtxConnectorTestResult {
|
||||
success: boolean;
|
||||
error?: string;
|
||||
/**
|
||||
* The original error thrown by the driver, preserved unflattened so the
|
||||
* connection-test path can re-throw it. Keeping the real error object lets
|
||||
* telemetry record the driver's actual error class (e.g. `ConnectionError`)
|
||||
* and `.code` (e.g. `ELOGIN`) instead of collapsing every failure to `Error`.
|
||||
*/
|
||||
cause?: unknown;
|
||||
}
|
||||
|
||||
/**
|
||||
* Single source of truth for a failed connector test result. Captures the
|
||||
* driver's message for display while preserving the original error as `cause`
|
||||
* so callers can surface its real class and code.
|
||||
*/
|
||||
export function connectorTestFailure(error: unknown): KtxConnectorTestResult {
|
||||
return {
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
cause: error,
|
||||
};
|
||||
}
|
||||
|
||||
export interface KtxScanConnector {
|
||||
|
|
|
|||
|
|
@ -78,7 +78,6 @@ export interface KtxIngestDeps {
|
|||
readReportFile?: typeof readIngestReportSnapshotFile;
|
||||
renderStoredMemoryFlow?: typeof renderMemoryFlowTui;
|
||||
startLiveMemoryFlow?: typeof startLiveMemoryFlowTui;
|
||||
abortSignal?: AbortSignal;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
localIngestOptions?: Pick<
|
||||
RunLocalIngestOptions,
|
||||
|
|
@ -94,23 +93,6 @@ export interface KtxIngestDeps {
|
|||
runtimeIo?: KtxIngestIo;
|
||||
}
|
||||
|
||||
function createCliAbortSignal(): { signal: AbortSignal; dispose: () => void } {
|
||||
const controller = new AbortController();
|
||||
let interrupted = false;
|
||||
const onSigint = () => {
|
||||
if (interrupted) {
|
||||
process.exit(130);
|
||||
}
|
||||
interrupted = true;
|
||||
controller.abort(new DOMException('Aborted', 'AbortError'));
|
||||
};
|
||||
process.on('SIGINT', onSigint);
|
||||
return {
|
||||
signal: controller.signal,
|
||||
dispose: () => process.off('SIGINT', onSigint),
|
||||
};
|
||||
}
|
||||
|
||||
const REPORT_SOURCE_LABELS = new Map<string, string>([
|
||||
['live-database', 'Database schema'],
|
||||
['historic-sql', 'Query history'],
|
||||
|
|
@ -382,12 +364,6 @@ function plainIngestEventProgress(
|
|||
message: event.message,
|
||||
...(event.transient !== undefined ? { transient: event.transient } : {}),
|
||||
};
|
||||
case 'rate_limit_wait':
|
||||
return {
|
||||
percent: 50,
|
||||
message: `Rate-limited (${event.provider}${event.rateLimitType ? ` ${event.rateLimitType}` : ''}); resuming in ${Math.ceil(event.remainingMs / 1_000)}s`,
|
||||
transient: true,
|
||||
};
|
||||
case 'work_unit_started': {
|
||||
const total = plannedWorkUnitCountThrough(snapshot, eventIndex);
|
||||
const ordinal = workUnitOrdinalThrough(snapshot, eventIndex, event.unitKey);
|
||||
|
|
@ -774,8 +750,6 @@ export async function runKtxIngest(
|
|||
);
|
||||
plainProgress?.start();
|
||||
structuredProgress?.start();
|
||||
const cliAbort = deps.abortSignal ? null : createCliAbortSignal();
|
||||
const abortSignal = deps.abortSignal ?? cliAbort?.signal;
|
||||
let result: LocalMetabaseFanoutResult;
|
||||
try {
|
||||
result = await executeMetabaseFanout({
|
||||
|
|
@ -789,7 +763,6 @@ export async function runKtxIngest(
|
|||
embeddingProvider,
|
||||
...(memoryFlow ? { memoryFlow } : {}),
|
||||
...(progress ? { progress } : {}),
|
||||
...(abortSignal ? { abortSignal } : {}),
|
||||
});
|
||||
plainProgress?.flush();
|
||||
if (args.outputMode === 'json') {
|
||||
|
|
@ -799,7 +772,6 @@ export async function runKtxIngest(
|
|||
}
|
||||
} finally {
|
||||
plainProgress?.flush();
|
||||
cliAbort?.dispose();
|
||||
}
|
||||
return result.status === 'all_failed' ? 1 : 0;
|
||||
}
|
||||
|
|
@ -848,8 +820,6 @@ export async function runKtxIngest(
|
|||
|
||||
plainProgress?.start();
|
||||
structuredProgress?.start();
|
||||
const cliAbort = deps.abortSignal ? null : createCliAbortSignal();
|
||||
const abortSignal = deps.abortSignal ?? cliAbort?.signal;
|
||||
|
||||
try {
|
||||
const result = await executeLocalIngest({
|
||||
|
|
@ -866,7 +836,6 @@ export async function runKtxIngest(
|
|||
embeddingProvider,
|
||||
...(args.debugLlmRequestFile ? { llmDebugRequestFile: args.debugLlmRequestFile } : {}),
|
||||
...(memoryFlow ? { memoryFlow } : {}),
|
||||
...(abortSignal ? { abortSignal } : {}),
|
||||
});
|
||||
if (shouldUseLiveViz && memoryFlow) {
|
||||
latestMemoryFlowSnapshot = finalRunMemoryFlowInput(memoryFlow.snapshot(), result.report);
|
||||
|
|
@ -885,7 +854,6 @@ export async function runKtxIngest(
|
|||
} finally {
|
||||
plainProgress?.flush();
|
||||
liveTui?.close();
|
||||
cliAbort?.dispose();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -23,8 +23,7 @@ import type { KtxScanArgs, KtxScanDeps } from './scan.js';
|
|||
import type { KtxTableRef } from './context/scan/types.js';
|
||||
import { profileMark } from './startup-profile.js';
|
||||
import { isDemoConnection } from './telemetry/demo-detect.js';
|
||||
import { emitProjectStackSnapshot, emitTelemetryEvent, reportException } from './telemetry/index.js';
|
||||
import { collectTelemetryRedactionSecrets } from './telemetry/redaction-secrets.js';
|
||||
import { emitProjectStackSnapshot, emitTelemetryEvent } from './telemetry/index.js';
|
||||
import { formatErrorDetail } from './telemetry/scrubber.js';
|
||||
|
||||
profileMark('module:public-ingest');
|
||||
|
|
@ -1120,63 +1119,30 @@ export async function runKtxPublicIngest(
|
|||
feature,
|
||||
});
|
||||
} catch (error) {
|
||||
await reportException({
|
||||
error,
|
||||
context: { source: 'ingest runtime', handled: true, fatal: false },
|
||||
projectDir: args.projectDir,
|
||||
io,
|
||||
redactionSecrets: await collectTelemetryRedactionSecrets({
|
||||
project,
|
||||
projectDir: args.projectDir,
|
||||
connectionId: args.targetConnectionId,
|
||||
includeLlm: true,
|
||||
includeEmbeddings: true,
|
||||
env: deps.env ?? process.env,
|
||||
}),
|
||||
});
|
||||
io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
const { runContextBuild } = await import('./context-build-view.js');
|
||||
const contextBuild = deps.runContextBuild ?? runContextBuild;
|
||||
try {
|
||||
const result = await contextBuild(
|
||||
project,
|
||||
{
|
||||
projectDir: args.projectDir,
|
||||
...(args.targetConnectionId ? { targetConnectionId: args.targetConnectionId } : {}),
|
||||
all: args.all,
|
||||
entrypoint: 'ingest',
|
||||
inputMode: args.inputMode,
|
||||
...(args.queryHistory ? { queryHistory: args.queryHistory } : {}),
|
||||
...(args.queryHistoryWindowDays !== undefined ? { queryHistoryWindowDays: args.queryHistoryWindowDays } : {}),
|
||||
...(args.scanMode ? { scanMode: args.scanMode } : {}),
|
||||
...(args.detectRelationships !== undefined ? { detectRelationships: args.detectRelationships } : {}),
|
||||
...(args.cliVersion ? { cliVersion: args.cliVersion } : {}),
|
||||
...(args.runtimeInstallPolicy ? { runtimeInstallPolicy: args.runtimeInstallPolicy } : {}),
|
||||
},
|
||||
io,
|
||||
);
|
||||
return result.exitCode;
|
||||
} catch (error) {
|
||||
await reportException({
|
||||
error,
|
||||
context: { source: 'ingest context-build', handled: true, fatal: false },
|
||||
const result = await contextBuild(
|
||||
project,
|
||||
{
|
||||
projectDir: args.projectDir,
|
||||
io,
|
||||
redactionSecrets: await collectTelemetryRedactionSecrets({
|
||||
project,
|
||||
projectDir: args.projectDir,
|
||||
connectionId: args.targetConnectionId,
|
||||
includeLlm: true,
|
||||
includeEmbeddings: true,
|
||||
env: deps.env ?? process.env,
|
||||
}),
|
||||
});
|
||||
io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
|
||||
return 1;
|
||||
}
|
||||
...(args.targetConnectionId ? { targetConnectionId: args.targetConnectionId } : {}),
|
||||
all: args.all,
|
||||
entrypoint: 'ingest',
|
||||
inputMode: args.inputMode,
|
||||
...(args.queryHistory ? { queryHistory: args.queryHistory } : {}),
|
||||
...(args.queryHistoryWindowDays !== undefined ? { queryHistoryWindowDays: args.queryHistoryWindowDays } : {}),
|
||||
...(args.scanMode ? { scanMode: args.scanMode } : {}),
|
||||
...(args.detectRelationships !== undefined ? { detectRelationships: args.detectRelationships } : {}),
|
||||
...(args.cliVersion ? { cliVersion: args.cliVersion } : {}),
|
||||
...(args.runtimeInstallPolicy ? { runtimeInstallPolicy: args.runtimeInstallPolicy } : {}),
|
||||
},
|
||||
io,
|
||||
);
|
||||
return result.exitCode;
|
||||
}
|
||||
|
||||
const plan = buildPublicIngestPlan(project, args);
|
||||
|
|
|
|||
|
|
@ -1,93 +0,0 @@
|
|||
import { styleText } from 'node:util';
|
||||
import { PasswordPrompt, type PasswordOptions } from '@clack/core';
|
||||
import { S_BAR, S_BAR_END, S_PASSWORD_MASK, settings, symbol } from '@clack/prompts';
|
||||
|
||||
// How many trailing characters of a pasted secret to leave visible so the user
|
||||
// can confirm what landed (e.g. `••••••a1b2`). Kept small on purpose.
|
||||
const REVEAL_TAIL_COUNT = 4;
|
||||
|
||||
/**
|
||||
* Mask every character of `userInput` except the last `tail`, but only reveal the
|
||||
* tail once the secret is long enough that the hidden portion still dominates
|
||||
* (`length > tail * 2`). Short secrets stay fully masked so we never expose most
|
||||
* of a small value. The returned string keeps the same code-unit length as the
|
||||
* input so clack's cursor slicing in `userInputWithCursor` stays aligned.
|
||||
*
|
||||
* @internal
|
||||
*/
|
||||
export function maskRevealingTail(userInput: string, maskChar: string, tail: number): string {
|
||||
const revealLength = userInput.length > tail * 2 ? tail : 0;
|
||||
const hiddenLength = userInput.length - revealLength;
|
||||
return maskChar.repeat(hiddenLength) + userInput.slice(hiddenLength);
|
||||
}
|
||||
|
||||
class RevealTailPasswordPrompt extends PasswordPrompt {
|
||||
readonly #maskChar: string;
|
||||
readonly #tail: number;
|
||||
|
||||
constructor(options: PasswordOptions & { tail: number }) {
|
||||
super(options);
|
||||
this.#maskChar = options.mask ?? S_PASSWORD_MASK;
|
||||
this.#tail = options.tail;
|
||||
}
|
||||
|
||||
override get masked(): string {
|
||||
return maskRevealingTail(this.userInput, this.#maskChar, this.#tail);
|
||||
}
|
||||
}
|
||||
|
||||
// Reproduces the @clack/prompts password frame (pinned to the installed version)
|
||||
// so this prompt is visually identical to every other setup prompt; the only
|
||||
// behavioral change is the tail-revealing `masked` getter above.
|
||||
function renderPasswordFrame(prompt: Omit<PasswordPrompt, 'prompt'>, message: string): string {
|
||||
const withGuide = settings.withGuide;
|
||||
const title = `${withGuide ? `${styleText('gray', S_BAR)}\n` : ''}${symbol(prompt.state)} ${message}\n`;
|
||||
const masked = prompt.masked;
|
||||
switch (prompt.state) {
|
||||
case 'error': {
|
||||
const bar = withGuide ? `${styleText('yellow', S_BAR)} ` : '';
|
||||
const end = withGuide ? `${styleText('yellow', S_BAR_END)} ` : '';
|
||||
return `${title.trim()}\n${bar}${masked}\n${end}${styleText('yellow', prompt.error)}\n`;
|
||||
}
|
||||
case 'submit': {
|
||||
const bar = withGuide ? `${styleText('gray', S_BAR)} ` : '';
|
||||
return `${title}${bar}${masked ? styleText('dim', masked) : ''}`;
|
||||
}
|
||||
case 'cancel': {
|
||||
const bar = withGuide ? `${styleText('gray', S_BAR)} ` : '';
|
||||
const body = masked ? styleText(['strikethrough', 'dim'], masked) : '';
|
||||
return `${title}${bar}${body}${masked && withGuide ? `\n${styleText('gray', S_BAR)}` : ''}`;
|
||||
}
|
||||
default: {
|
||||
const bar = withGuide ? `${styleText('cyan', S_BAR)} ` : '';
|
||||
const end = withGuide ? styleText('cyan', S_BAR_END) : '';
|
||||
return `${title}${bar}${prompt.userInputWithCursor}\n${end}\n`;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export interface RevealPasswordOptions {
|
||||
message: string;
|
||||
mask?: string;
|
||||
tail?: number;
|
||||
validate?: PasswordOptions['validate'];
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
/**
|
||||
* Drop-in replacement for clack's `password()` that reveals the last few
|
||||
* characters of the entered value while typing. Resolves to the raw value or the
|
||||
* clack cancel symbol, matching `password()`'s contract.
|
||||
*/
|
||||
export function revealPassword(options: RevealPasswordOptions): Promise<string | symbol> {
|
||||
const prompt = new RevealTailPasswordPrompt({
|
||||
mask: options.mask ?? S_PASSWORD_MASK,
|
||||
tail: options.tail ?? REVEAL_TAIL_COUNT,
|
||||
validate: options.validate,
|
||||
signal: options.signal,
|
||||
render() {
|
||||
return renderPasswordFrame(this, options.message);
|
||||
},
|
||||
});
|
||||
return prompt.prompt() as Promise<string | symbol>;
|
||||
}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
import type { KtxProgressPort, KtxScanMode, KtxScanReport, KtxScanWarning } from './context/scan/types.js';
|
||||
import { runLocalScan } from './context/scan/local-scan.js';
|
||||
import { loadKtxProject, type KtxLocalProject } from './context/project/project.js';
|
||||
import { loadKtxProject } from './context/project/project.js';
|
||||
import { getKtxCliPackageInfo } from './cli-runtime.js';
|
||||
import { resolveProjectEmbeddingProvider } from './embedding-resolution.js';
|
||||
import type { KtxCliIo } from './index.js';
|
||||
|
|
@ -8,8 +8,7 @@ import { createKtxCliLocalIngestAdapters } from './local-adapters.js';
|
|||
import { createKtxCliScanConnector } from './local-scan-connectors.js';
|
||||
import type { KtxManagedPythonInstallPolicy } from './managed-python-command.js';
|
||||
import { profileMark } from './startup-profile.js';
|
||||
import { emitTelemetryEvent, reportException } from './telemetry/index.js';
|
||||
import { collectTelemetryRedactionSecrets } from './telemetry/redaction-secrets.js';
|
||||
import { emitTelemetryEvent } from './telemetry/index.js';
|
||||
import { formatErrorDetail, scrubErrorClass } from './telemetry/scrubber.js';
|
||||
|
||||
profileMark('module:scan');
|
||||
|
|
@ -323,9 +322,8 @@ export function createCliScanProgress(
|
|||
|
||||
export async function runKtxScan(args: KtxScanArgs, io: KtxCliIo = process, deps: KtxScanDeps = {}): Promise<number> {
|
||||
const startedAt = performance.now();
|
||||
let project: KtxLocalProject | undefined;
|
||||
try {
|
||||
project = await loadKtxProject({ projectDir: args.projectDir });
|
||||
const project = await loadKtxProject({ projectDir: args.projectDir });
|
||||
const resolveEmbeddingProvider = deps.resolveEmbeddingProvider ?? resolveProjectEmbeddingProvider;
|
||||
const resolution = await resolveEmbeddingProvider(project, {
|
||||
mode: 'ensure',
|
||||
|
|
@ -399,20 +397,6 @@ export async function runKtxScan(args: KtxScanArgs, io: KtxCliIo = process, deps
|
|||
...(errorDetail ? { errorDetail } : {}),
|
||||
},
|
||||
});
|
||||
await reportException({
|
||||
error,
|
||||
context: { source: 'scan run', handled: true, fatal: false },
|
||||
projectDir: args.projectDir,
|
||||
io,
|
||||
redactionSecrets: await collectTelemetryRedactionSecrets({
|
||||
project,
|
||||
projectDir: args.projectDir,
|
||||
connectionId: args.connectionId,
|
||||
includeLlm: true,
|
||||
includeEmbeddings: true,
|
||||
env: process.env,
|
||||
}),
|
||||
});
|
||||
io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
|
||||
return 1;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -73,7 +73,6 @@ export type KtxSetupDatabaseDriver =
|
|||
export interface KtxSetupDatabasesArgs {
|
||||
projectDir: string;
|
||||
inputMode: 'auto' | 'disabled';
|
||||
debug?: boolean;
|
||||
yes?: boolean;
|
||||
cliVersion?: string;
|
||||
runtimeInstallPolicy?: KtxManagedPythonInstallPolicy;
|
||||
|
|
@ -1627,12 +1626,7 @@ function hasServiceAccountsBlock(connection: KtxProjectConnectionConfig | undefi
|
|||
return 'serviceAccounts' in filters;
|
||||
}
|
||||
|
||||
function printQueryHistoryFilterProposal(io: KtxCliIo, proposal: QueryHistoryFilterProposal, debug = false): void {
|
||||
if (debug && proposal.parseFailedTemplateIds.length > 0) {
|
||||
io.stderr.write(
|
||||
`[debug] query-history filter picker could not parse ${proposal.parseFailedTemplateIds.length} template(s): ${proposal.parseFailedTemplateIds.join(', ')}\n`,
|
||||
);
|
||||
}
|
||||
function printQueryHistoryFilterProposal(io: KtxCliIo, proposal: QueryHistoryFilterProposal): void {
|
||||
if (proposal.excludedRoles.length === 0) {
|
||||
if (proposal.skipped?.reason === 'no-llm') {
|
||||
io.stdout.write('│ Query-history filter picker skipped: no LLM is configured.\n');
|
||||
|
|
@ -1641,12 +1635,6 @@ function printQueryHistoryFilterProposal(io: KtxCliIo, proposal: QueryHistoryFil
|
|||
} else if (proposal.skipped?.reason === 'no-in-scope-history') {
|
||||
io.stdout.write('│ Query-history filter picker found no in-scope service-account exclusions.\n');
|
||||
}
|
||||
if (proposal.parseFailedTemplateIds.length > 0) {
|
||||
const count = proposal.parseFailedTemplateIds.length;
|
||||
io.stdout.write(
|
||||
`│ Skipped ${count} query template${count === 1 ? '' : 's'} ktx could not parse (run with --debug to list them).\n`,
|
||||
);
|
||||
}
|
||||
for (const warning of proposal.warnings) {
|
||||
io.stdout.write(`│ ! ${warning}\n`);
|
||||
}
|
||||
|
|
@ -1739,17 +1727,12 @@ async function maybeProposeQueryHistoryFilters(input: {
|
|||
deps: input.deps,
|
||||
});
|
||||
if (!llmRuntime && !input.deps.queryHistoryFilterPicker) {
|
||||
printQueryHistoryFilterProposal(
|
||||
input.io,
|
||||
{
|
||||
excludedRoles: [],
|
||||
consideredRoleCount: 0,
|
||||
skipped: { reason: 'no-llm' },
|
||||
warnings: [],
|
||||
parseFailedTemplateIds: [],
|
||||
},
|
||||
input.args.debug === true,
|
||||
);
|
||||
printQueryHistoryFilterProposal(input.io, {
|
||||
excludedRoles: [],
|
||||
consideredRoleCount: 0,
|
||||
skipped: { reason: 'no-llm' },
|
||||
warnings: [],
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -1790,19 +1773,7 @@ async function maybeProposeQueryHistoryFilters(input: {
|
|||
userServiceAccountsPresent,
|
||||
});
|
||||
|
||||
printQueryHistoryFilterProposal(input.io, proposal, input.args.debug === true);
|
||||
await emitTelemetryEvent({
|
||||
name: 'query_history_filter_completed',
|
||||
projectDir: input.projectDir,
|
||||
io: input.io,
|
||||
fields: {
|
||||
dialect,
|
||||
consideredRoleCount: proposal.consideredRoleCount,
|
||||
excludedRoleCount: proposal.excludedRoles.length,
|
||||
parseFailedCount: proposal.parseFailedTemplateIds.length,
|
||||
outcome: 'ok',
|
||||
},
|
||||
});
|
||||
printQueryHistoryFilterProposal(input.io, proposal);
|
||||
if (proposal.skipped?.reason === 'user-block-present') {
|
||||
input.io.stdout.write('│ Existing query-history service-account filters left unchanged.\n');
|
||||
return;
|
||||
|
|
|
|||
|
|
@ -222,8 +222,8 @@ async function chooseCredentialRef(
|
|||
const choice = await prompts.select({
|
||||
message: `How should KTX find your ${embeddingBackendDisplayName(backend)} embedding API key?`,
|
||||
options: [
|
||||
{ value: 'paste', label: 'Paste a key and save it as a local secret file' },
|
||||
{ value: 'env', label: `Use ${defaultEnv} from the environment` },
|
||||
{ value: 'paste', label: 'Paste a key and save it as a local secret file' },
|
||||
{ value: 'back', label: 'Back' },
|
||||
],
|
||||
});
|
||||
|
|
|
|||
|
|
@ -470,8 +470,8 @@ async function chooseCredentialRef(
|
|||
const choice = await prompts.select({
|
||||
message: `How should KTX find your Anthropic API key?\n\n${ANTHROPIC_CREDENTIAL_PROMPT_CONTEXT}`,
|
||||
options: [
|
||||
{ value: 'paste', label: 'Paste a key and save it as a local secret file' },
|
||||
{ value: 'env', label: 'Use ANTHROPIC_API_KEY from the environment' },
|
||||
{ value: 'paste', label: 'Paste a key and save it as a local secret file' },
|
||||
{ value: 'back', label: 'Back' },
|
||||
],
|
||||
});
|
||||
|
|
|
|||
|
|
@ -9,12 +9,12 @@ import {
|
|||
log,
|
||||
multiselect,
|
||||
note,
|
||||
password,
|
||||
select,
|
||||
text,
|
||||
} from '@clack/prompts';
|
||||
import type { KtxCliIo } from './cli-runtime.js';
|
||||
import { withMenuOptionsSpacing, withTextInputNavigation } from './prompt-navigation.js';
|
||||
import { revealPassword } from './reveal-password-prompt.js';
|
||||
import { withSetupInterruptConfirmation } from './setup-interrupt.js';
|
||||
|
||||
export interface KtxSetupPromptOption<Value extends string = string> {
|
||||
|
|
@ -189,7 +189,7 @@ export function createKtxSetupPromptAdapter(options: KtxSetupPromptAdapterOption
|
|||
},
|
||||
async password(promptOptions) {
|
||||
const value = await withSetupInterruptConfirmation(() =>
|
||||
revealPassword({ ...promptOptions, message: withTextInputNavigation(promptOptions.message) }),
|
||||
password({ ...promptOptions, message: withTextInputNavigation(promptOptions.message) }),
|
||||
);
|
||||
return isCancel(value) ? undefined : String(value);
|
||||
},
|
||||
|
|
|
|||
|
|
@ -119,11 +119,11 @@ export interface KtxSetupSourcesDeps {
|
|||
|
||||
const SOURCE_OPTIONS: Array<{ value: KtxSetupSourceType; label: string }> = [
|
||||
{ value: 'dbt', label: 'dbt' },
|
||||
{ value: 'metabase', label: 'Metabase' },
|
||||
{ value: 'notion', label: 'Notion' },
|
||||
{ value: 'metricflow', label: 'MetricFlow' },
|
||||
{ value: 'metabase', label: 'Metabase' },
|
||||
{ value: 'looker', label: 'Looker' },
|
||||
{ value: 'lookml', label: 'LookML' },
|
||||
{ value: 'notion', label: 'Notion' },
|
||||
];
|
||||
|
||||
const SOURCE_LABELS = Object.fromEntries(SOURCE_OPTIONS.map((option) => [option.value, option.label])) as Record<
|
||||
|
|
@ -269,8 +269,8 @@ async function chooseSourceCredentialRef(input: {
|
|||
message: `How should KTX find your ${input.label}?`,
|
||||
options: [
|
||||
...(input.existingRef ? [{ value: 'keep', label: 'Keep existing credential' }] : []),
|
||||
{ value: 'paste', label: 'Paste a key and save it as a local secret file' },
|
||||
{ value: 'env', label: `Use ${input.envName} from the environment` },
|
||||
{ value: 'paste', label: 'Paste a key and save it as a local secret file' },
|
||||
{ value: 'back', label: 'Back' },
|
||||
],
|
||||
});
|
||||
|
|
@ -307,8 +307,8 @@ async function chooseGitAuthCredentialRef(input: {
|
|||
message: `${label} repo requires authentication.`,
|
||||
options: [
|
||||
...(input.existingRef ? [{ value: 'keep', label: 'Keep existing credential' }] : []),
|
||||
{ value: 'paste', label: 'Paste a token and save it as a local secret file' },
|
||||
{ value: 'env', label: 'Use GITHUB_TOKEN from the environment' },
|
||||
{ value: 'paste', label: 'Paste a token and save it as a local secret file' },
|
||||
{ value: 'skip', label: 'Skip — try without authentication' },
|
||||
{ value: 'back', label: 'Back' },
|
||||
],
|
||||
|
|
@ -1063,8 +1063,8 @@ async function promptForInteractiveSource(
|
|||
const selectedLocation = await prompts.select({
|
||||
message: `${source} source location`,
|
||||
options: [
|
||||
{ value: 'git', label: 'Git URL' },
|
||||
{ value: 'path', label: 'Local path' },
|
||||
{ value: 'git', label: 'Git URL' },
|
||||
{ value: 'back', label: 'Back' },
|
||||
],
|
||||
});
|
||||
|
|
@ -1343,8 +1343,8 @@ async function promptForInteractiveSource(
|
|||
const crawlMode = await prompts.select({
|
||||
message: 'Which Notion pages should KTX ingest?',
|
||||
options: [
|
||||
{ value: 'all_accessible', label: 'All pages the integration can access' },
|
||||
{ value: 'selected_roots', label: 'Specific pages and their subpages (choose them in a picker)' },
|
||||
{ value: 'all_accessible', label: 'All pages the integration can access' },
|
||||
{ value: 'back', label: 'Back' },
|
||||
],
|
||||
});
|
||||
|
|
@ -2064,7 +2064,7 @@ export async function runKtxSetupSourcesStep(
|
|||
const addMore = await prompts.select({
|
||||
message: `${readyConnectionIds.length} context source${readyConnectionIds.length > 1 ? 's' : ''} configured (${readyConnectionIds.join(', ')}). Add another?`,
|
||||
options: [
|
||||
{ value: 'done', label: 'Done adding context sources' },
|
||||
{ value: 'done', label: 'Done — continue to context build' },
|
||||
{ value: 'edit', label: 'Edit an existing context source' },
|
||||
{ value: 'add', label: 'Add another context source' },
|
||||
],
|
||||
|
|
|
|||
|
|
@ -80,7 +80,6 @@ export type KtxSetupArgs =
|
|||
agentScope?: KtxAgentScope;
|
||||
skipAgents?: boolean;
|
||||
inputMode: 'auto' | 'disabled';
|
||||
debug?: boolean;
|
||||
yes: boolean;
|
||||
cliVersion: string;
|
||||
llmBackend?: KtxSetupLlmBackend;
|
||||
|
|
@ -736,7 +735,6 @@ async function runKtxSetupInner(args: KtxSetupArgs, io: KtxCliIo, deps: KtxSetup
|
|||
{
|
||||
projectDir: projectResult.projectDir,
|
||||
inputMode: args.inputMode,
|
||||
...(args.debug !== undefined ? { debug: args.debug } : {}),
|
||||
yes: args.yes,
|
||||
cliVersion: args.cliVersion,
|
||||
runtimeInstallPolicy: setupRuntimeInstallPolicy(args),
|
||||
|
|
|
|||
|
|
@ -26,8 +26,7 @@ import {
|
|||
type KtxManagedPythonInstallPolicy,
|
||||
} from './managed-python-command.js';
|
||||
import { profileMark } from './startup-profile.js';
|
||||
import { emitTelemetryEvent, reportException } from './telemetry/index.js';
|
||||
import { collectTelemetryRedactionSecrets } from './telemetry/redaction-secrets.js';
|
||||
import { emitTelemetryEvent } from './telemetry/index.js';
|
||||
import { scrubErrorClass } from './telemetry/scrubber.js';
|
||||
|
||||
profileMark('module:sl');
|
||||
|
|
@ -203,9 +202,8 @@ function ambiguousSourceMessage(sourceName: string, connectionIds: readonly stri
|
|||
export async function runKtxSl(args: KtxSlArgs, io: KtxSlIo = process, deps: KtxSlDeps = {}): Promise<number> {
|
||||
const startedAt = performance.now();
|
||||
let queryForTelemetry: SemanticLayerQueryInput | undefined;
|
||||
let project: KtxLocalProject | undefined;
|
||||
try {
|
||||
project = await (deps.loadProject ?? loadKtxProject)({ projectDir: args.projectDir });
|
||||
const project = await (deps.loadProject ?? loadKtxProject)({ projectDir: args.projectDir });
|
||||
if (args.command === 'list') {
|
||||
const sources = await listLocalSlSources(project, { connectionId: args.connectionId });
|
||||
await printSlSources({
|
||||
|
|
@ -322,7 +320,7 @@ export async function runKtxSl(args: KtxSlArgs, io: KtxSlIo = process, deps: Ktx
|
|||
projectDir: args.projectDir,
|
||||
});
|
||||
const queryExecutor = args.execute ? (deps.createQueryExecutor ?? createDefaultLocalQueryExecutor)() : undefined;
|
||||
const result = await compileLocalSlQuery(project, {
|
||||
const result = await compileLocalSlQuery(project as KtxLocalProject, {
|
||||
connectionId: args.connectionId,
|
||||
query,
|
||||
compute,
|
||||
|
|
@ -353,20 +351,6 @@ export async function runKtxSl(args: KtxSlArgs, io: KtxSlIo = process, deps: Ktx
|
|||
const _exhaustive: never = args;
|
||||
throw new Error(`Unsupported sl command: ${JSON.stringify(_exhaustive)}`);
|
||||
} catch (error) {
|
||||
await reportException({
|
||||
error,
|
||||
context: { source: `sl ${args.command}`, handled: true, fatal: false },
|
||||
projectDir: args.projectDir,
|
||||
io,
|
||||
redactionSecrets: await collectTelemetryRedactionSecrets({
|
||||
project,
|
||||
projectDir: args.projectDir,
|
||||
connectionId: args.connectionId,
|
||||
includeLlm: args.command === 'query',
|
||||
includeEmbeddings: args.command === 'search' || args.command === 'query',
|
||||
env: process.env,
|
||||
}),
|
||||
});
|
||||
if (args.command === 'validate') {
|
||||
const errorClass = scrubErrorClass(error);
|
||||
await emitTelemetryEvent({
|
||||
|
|
|
|||
|
|
@ -7,8 +7,7 @@ import { createKtxCliScanConnector } from './local-scan-connectors.js';
|
|||
import { createManagedDaemonSqlAnalysisPort } from './managed-python-http.js';
|
||||
import { profileMark } from './startup-profile.js';
|
||||
import { isDemoConnection } from './telemetry/demo-detect.js';
|
||||
import { emitTelemetryEvent, reportException } from './telemetry/index.js';
|
||||
import { collectTelemetryRedactionSecrets } from './telemetry/redaction-secrets.js';
|
||||
import { emitTelemetryEvent } from './telemetry/index.js';
|
||||
import { scrubErrorClass } from './telemetry/scrubber.js';
|
||||
|
||||
profileMark('module:sql');
|
||||
|
|
@ -143,9 +142,8 @@ export async function runKtxSql(args: KtxSqlArgs, io: KtxCliIo = process, deps:
|
|||
const startedAt = performance.now();
|
||||
let driver = 'unknown';
|
||||
let demoConnection = false;
|
||||
let project: KtxLocalProject | undefined;
|
||||
try {
|
||||
project = await (deps.loadProject ?? loadKtxProject)({ projectDir: args.projectDir });
|
||||
const project = await (deps.loadProject ?? loadKtxProject)({ projectDir: args.projectDir });
|
||||
const connection = project.config.connections[args.connectionId];
|
||||
if (!connection) {
|
||||
throw new Error(`Connection "${args.connectionId}" is not configured in ktx.yaml`);
|
||||
|
|
@ -173,7 +171,7 @@ export async function runKtxSql(args: KtxSqlArgs, io: KtxCliIo = process, deps:
|
|||
const createScanConnector = deps.createScanConnector ?? createKtxCliScanConnector;
|
||||
let connector: KtxScanConnector | null = null;
|
||||
try {
|
||||
connector = await createScanConnector(project, args.connectionId);
|
||||
connector = await createScanConnector(project as KtxLocalProject, args.connectionId);
|
||||
if (!connector.capabilities.readOnlySql || !connector.executeReadOnly) {
|
||||
throw new Error(`Connection "${args.connectionId}" does not support read-only SQL execution.`);
|
||||
}
|
||||
|
|
@ -220,20 +218,6 @@ export async function runKtxSql(args: KtxSqlArgs, io: KtxCliIo = process, deps:
|
|||
...(errorClass ? { errorClass } : {}),
|
||||
},
|
||||
});
|
||||
await reportException({
|
||||
error,
|
||||
context: { source: 'sql run', handled: true, fatal: false },
|
||||
projectDir: args.projectDir,
|
||||
io,
|
||||
redactionSecrets: await collectTelemetryRedactionSecrets({
|
||||
project,
|
||||
projectDir: args.projectDir,
|
||||
connectionId: args.connectionId,
|
||||
includeLlm: false,
|
||||
includeEmbeddings: false,
|
||||
env: process.env,
|
||||
}),
|
||||
});
|
||||
io.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`);
|
||||
return 1;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,16 +16,6 @@ type PostHogClient = {
|
|||
properties: Record<string, unknown>;
|
||||
groups?: Record<string, string>;
|
||||
}): void;
|
||||
captureException(
|
||||
error: unknown,
|
||||
distinctId?: string,
|
||||
additionalProperties?: Record<string, unknown>,
|
||||
): void;
|
||||
captureExceptionImmediate(
|
||||
error: unknown,
|
||||
distinctId?: string,
|
||||
additionalProperties?: Record<string, unknown>,
|
||||
): Promise<void>;
|
||||
shutdown(): Promise<void> | void;
|
||||
};
|
||||
|
||||
|
|
@ -115,57 +105,6 @@ export async function trackTelemetryEvent(input: {
|
|||
}
|
||||
}
|
||||
|
||||
function writeDebugExceptionPayload(input: {
|
||||
error: Error;
|
||||
distinctId: string;
|
||||
properties: Record<string, unknown>;
|
||||
stderr: TelemetrySink;
|
||||
}): void {
|
||||
input.stderr.write(
|
||||
`[telemetry-exception] ${JSON.stringify({
|
||||
distinctId: input.distinctId,
|
||||
message: input.error.message,
|
||||
name: input.error.name,
|
||||
properties: input.properties,
|
||||
})}\n`,
|
||||
);
|
||||
}
|
||||
|
||||
export async function trackTelemetryException(input: {
|
||||
error: Error;
|
||||
distinctId: string;
|
||||
properties: Record<string, unknown>;
|
||||
env?: TelemetryEmitterEnv;
|
||||
stderr: TelemetrySink;
|
||||
projectApiKey?: string;
|
||||
host?: string;
|
||||
immediate?: boolean;
|
||||
}): Promise<void> {
|
||||
const env = input.env ?? process.env;
|
||||
|
||||
if (debugEnabled(env)) {
|
||||
writeDebugExceptionPayload(input);
|
||||
return;
|
||||
}
|
||||
|
||||
const projectApiKey = telemetryProjectApiKey(input.projectApiKey);
|
||||
const host = telemetryHost(env, input.host);
|
||||
const client = await getPostHogClient(projectApiKey, host);
|
||||
if (!client) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
if (input.immediate) {
|
||||
await client.captureExceptionImmediate(input.error, input.distinctId, input.properties);
|
||||
return;
|
||||
}
|
||||
client.captureException(input.error, input.distinctId, input.properties);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
export async function shutdownTelemetryEmitter(): Promise<void> {
|
||||
const client = await clientPromise;
|
||||
if (!client) {
|
||||
|
|
|
|||
|
|
@ -206,17 +206,6 @@
|
|||
"errorClass",
|
||||
"durationMs"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "query_history_filter_completed",
|
||||
"description": "Emitted after the setup query-history service-account filter picker runs.",
|
||||
"fields": [
|
||||
"dialect",
|
||||
"consideredRoleCount",
|
||||
"excludedRoleCount",
|
||||
"parseFailedCount",
|
||||
"outcome"
|
||||
]
|
||||
}
|
||||
],
|
||||
"$defs": {
|
||||
|
|
@ -1445,77 +1434,6 @@
|
|||
"durationMs"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"query_history_filter_completed": {
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"cliVersion": {
|
||||
"type": "string"
|
||||
},
|
||||
"nodeVersion": {
|
||||
"type": "string"
|
||||
},
|
||||
"osPlatform": {
|
||||
"type": "string"
|
||||
},
|
||||
"osRelease": {
|
||||
"type": "string"
|
||||
},
|
||||
"arch": {
|
||||
"type": "string"
|
||||
},
|
||||
"runtime": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"node",
|
||||
"daemon-py"
|
||||
]
|
||||
},
|
||||
"isCi": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"dialect": {
|
||||
"type": "string"
|
||||
},
|
||||
"consideredRoleCount": {
|
||||
"type": "integer",
|
||||
"minimum": 0,
|
||||
"maximum": 9007199254740991
|
||||
},
|
||||
"excludedRoleCount": {
|
||||
"type": "integer",
|
||||
"minimum": 0,
|
||||
"maximum": 9007199254740991
|
||||
},
|
||||
"parseFailedCount": {
|
||||
"type": "integer",
|
||||
"minimum": 0,
|
||||
"maximum": 9007199254740991
|
||||
},
|
||||
"outcome": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"ok",
|
||||
"error"
|
||||
]
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"cliVersion",
|
||||
"nodeVersion",
|
||||
"osPlatform",
|
||||
"osRelease",
|
||||
"arch",
|
||||
"runtime",
|
||||
"isCi",
|
||||
"dialect",
|
||||
"consideredRoleCount",
|
||||
"excludedRoleCount",
|
||||
"parseFailedCount",
|
||||
"outcome"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -206,16 +206,6 @@ const sqlGenCompletedSchema = telemetryCommonEnvelopeSchema
|
|||
})
|
||||
.strict();
|
||||
|
||||
const queryHistoryFilterCompletedSchema = telemetryCommonEnvelopeSchema
|
||||
.extend({
|
||||
dialect: z.string(),
|
||||
consideredRoleCount: z.number().int().nonnegative(),
|
||||
excludedRoleCount: z.number().int().nonnegative(),
|
||||
parseFailedCount: z.number().int().nonnegative(),
|
||||
outcome: outcomeSchema,
|
||||
})
|
||||
.strict();
|
||||
|
||||
/** @internal */
|
||||
export const telemetryEventSchemas = {
|
||||
install_first_run: installFirstRunSchema,
|
||||
|
|
@ -235,7 +225,6 @@ export const telemetryEventSchemas = {
|
|||
daemon_stopped: daemonStoppedSchema,
|
||||
sl_plan_completed: slPlanCompletedSchema,
|
||||
sql_gen_completed: sqlGenCompletedSchema,
|
||||
query_history_filter_completed: queryHistoryFilterCompletedSchema,
|
||||
} as const;
|
||||
|
||||
/** @internal */
|
||||
|
|
@ -371,11 +360,6 @@ export const telemetryEventCatalog = [
|
|||
description: 'Emitted after daemon SQL generation completes.',
|
||||
fields: ['outcome', 'dialect', 'errorClass', 'durationMs'],
|
||||
},
|
||||
{
|
||||
name: 'query_history_filter_completed',
|
||||
description: 'Emitted after the setup query-history service-account filter picker runs.',
|
||||
fields: ['dialect', 'consideredRoleCount', 'excludedRoleCount', 'parseFailedCount', 'outcome'],
|
||||
},
|
||||
] as const;
|
||||
|
||||
export type TelemetryEventName = keyof typeof telemetryEventSchemas;
|
||||
|
|
|
|||
|
|
@ -1,201 +0,0 @@
|
|||
import { inspect } from 'node:util';
|
||||
|
||||
import { getKtxCliPackageInfo, type KtxCliIo, type KtxCliPackageInfo } from '../cli-runtime.js';
|
||||
import { buildCommonEnvelope } from './events.js';
|
||||
import { trackTelemetryException } from './emitter.js';
|
||||
import { computeTelemetryProjectId, loadTelemetryIdentity } from './identity.js';
|
||||
|
||||
export interface ExceptionContext {
|
||||
source: string;
|
||||
handled: boolean;
|
||||
fatal: boolean;
|
||||
extra?: Record<string, string | number | boolean>;
|
||||
}
|
||||
|
||||
type AnyObject = object;
|
||||
|
||||
const reportedObjects = new WeakSet<AnyObject>();
|
||||
const recentHandledPrimitives: string[] = [];
|
||||
const RECENT_PRIMITIVE_LIMIT = 128;
|
||||
|
||||
function primitiveKey(value: unknown): string {
|
||||
return `${typeof value}:${String(value)}`;
|
||||
}
|
||||
|
||||
function rememberHandledPrimitive(value: unknown): void {
|
||||
recentHandledPrimitives.push(primitiveKey(value));
|
||||
if (recentHandledPrimitives.length > RECENT_PRIMITIVE_LIMIT) {
|
||||
recentHandledPrimitives.splice(0, recentHandledPrimitives.length - RECENT_PRIMITIVE_LIMIT);
|
||||
}
|
||||
}
|
||||
|
||||
function consumeHandledPrimitive(value: unknown): boolean {
|
||||
const key = primitiveKey(value);
|
||||
const index = recentHandledPrimitives.indexOf(key);
|
||||
if (index < 0) {
|
||||
return false;
|
||||
}
|
||||
recentHandledPrimitives.splice(index, 1);
|
||||
return true;
|
||||
}
|
||||
|
||||
function shouldSkipAsAlreadyReported(error: unknown, handled: boolean): boolean {
|
||||
if ((typeof error === 'object' || typeof error === 'function') && error !== null) {
|
||||
if (reportedObjects.has(error)) {
|
||||
return true;
|
||||
}
|
||||
reportedObjects.add(error);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (handled) {
|
||||
rememberHandledPrimitive(error);
|
||||
return false;
|
||||
}
|
||||
|
||||
return consumeHandledPrimitive(error);
|
||||
}
|
||||
|
||||
function escapeRegExp(value: string): string {
|
||||
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
}
|
||||
|
||||
function redactStaticPatterns(value: string): string {
|
||||
return value
|
||||
.replace(/([a-z][a-z0-9+.-]*:\/\/[^:\s/@]+:)([^@\s/]+)(@)/gi, '$1[redacted]$3')
|
||||
.replace(/\b(password|pwd)=([^;&\s]+)/gi, '$1=[redacted]')
|
||||
.replace(/\bAuthorization\s*:\s*[^\r\n,;]+/gi, 'Authorization: [redacted]')
|
||||
.replace(/\bBearer\s+[A-Za-z0-9._~+/=-]+/gi, 'Bearer [redacted]')
|
||||
.replace(/\b(api[_-]?key)\s*[:=]\s*([^\s,;]+)/gi, '$1=[redacted]')
|
||||
.replace(/\b(KTX_[A-Z0-9_]*|[A-Z0-9_]*(?:TOKEN|SECRET))\s*[:=]\s*([^\s,;]+)/g, '$1=[redacted]')
|
||||
.replace(/([?&](?:X-Amz-Signature|X-Goog-Signature|sig)=)[^&\s]+/gi, '$1[redacted]');
|
||||
}
|
||||
|
||||
function redactText(value: string, secrets: ReadonlyArray<string>): string {
|
||||
let redacted = value;
|
||||
for (const secret of secrets) {
|
||||
if (secret) {
|
||||
redacted = redacted.replace(new RegExp(escapeRegExp(secret), 'g'), '[redacted]');
|
||||
}
|
||||
}
|
||||
return redactStaticPatterns(redacted);
|
||||
}
|
||||
|
||||
const FORBIDDEN_EXTRA_PROPERTY_KEYS = new Set([
|
||||
'argv',
|
||||
'args',
|
||||
'env',
|
||||
'environment',
|
||||
'sql',
|
||||
'query',
|
||||
'prompt',
|
||||
'mcparguments',
|
||||
'mcpargs',
|
||||
'tablename',
|
||||
'schemaname',
|
||||
'columnname',
|
||||
'databaseurl',
|
||||
'connectionstring',
|
||||
'url',
|
||||
'password',
|
||||
'token',
|
||||
'apikey',
|
||||
'api_key',
|
||||
'authorization',
|
||||
]);
|
||||
|
||||
function safeExtraProperties(
|
||||
extra: Record<string, string | number | boolean> | undefined,
|
||||
): Record<string, string | number | boolean> {
|
||||
const safe: Record<string, string | number | boolean> = {};
|
||||
for (const [key, value] of Object.entries(extra ?? {})) {
|
||||
if (!FORBIDDEN_EXTRA_PROPERTY_KEYS.has(key.replace(/[^a-z0-9_]/gi, '').toLowerCase())) {
|
||||
safe[key] = value;
|
||||
}
|
||||
}
|
||||
return safe;
|
||||
}
|
||||
|
||||
function toMessage(error: unknown): string {
|
||||
if (error instanceof Error) {
|
||||
return error.message;
|
||||
}
|
||||
if (typeof error === 'string') {
|
||||
return error;
|
||||
}
|
||||
return inspect(error, { depth: 4, breakLength: 120 });
|
||||
}
|
||||
|
||||
function sanitizedError(error: unknown, secrets: ReadonlyArray<string>): Error {
|
||||
if (error instanceof Error) {
|
||||
const cause = 'cause' in error ? (error as Error & { cause?: unknown }).cause : undefined;
|
||||
const clone = new Error(redactText(error.message, secrets), {
|
||||
...(cause !== undefined ? { cause: sanitizedError(cause, secrets) } : {}),
|
||||
});
|
||||
clone.name = error.name;
|
||||
if (error.stack) {
|
||||
clone.stack = redactText(error.stack, secrets);
|
||||
}
|
||||
return clone;
|
||||
}
|
||||
return new Error(redactText(toMessage(error), secrets));
|
||||
}
|
||||
|
||||
export async function reportException(input: {
|
||||
error: unknown;
|
||||
context: ExceptionContext;
|
||||
io: KtxCliIo;
|
||||
packageInfo?: KtxCliPackageInfo;
|
||||
projectDir?: string;
|
||||
immediate?: boolean;
|
||||
redactionSecrets?: ReadonlyArray<string>;
|
||||
}): Promise<void> {
|
||||
try {
|
||||
if (shouldSkipAsAlreadyReported(input.error, input.context.handled)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const debug = process.env.KTX_TELEMETRY_DEBUG === '1';
|
||||
const identity = await loadTelemetryIdentity({
|
||||
stderr: input.io.stderr,
|
||||
env: process.env,
|
||||
});
|
||||
|
||||
if ((!identity.enabled || !identity.installId) && !debug) {
|
||||
return;
|
||||
}
|
||||
|
||||
const packageInfo = input.packageInfo ?? getKtxCliPackageInfo();
|
||||
const installId = identity.installId ?? 'debug';
|
||||
const projectId = input.projectDir ? computeTelemetryProjectId(installId, input.projectDir) : undefined;
|
||||
const safeError = sanitizedError(input.error, input.redactionSecrets ?? []);
|
||||
const properties: Record<string, unknown> = {
|
||||
...buildCommonEnvelope({
|
||||
cliVersion: packageInfo.version,
|
||||
isCi: Boolean(process.env.CI),
|
||||
}),
|
||||
source: input.context.source,
|
||||
handled: input.context.handled,
|
||||
fatal: input.context.fatal,
|
||||
...(projectId ? { projectId } : {}),
|
||||
...safeExtraProperties(input.context.extra),
|
||||
};
|
||||
|
||||
delete properties.$groups;
|
||||
await trackTelemetryException({
|
||||
error: safeError,
|
||||
distinctId: installId,
|
||||
properties,
|
||||
env: process.env,
|
||||
stderr: input.io.stderr,
|
||||
immediate: input.immediate,
|
||||
});
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function __resetTelemetryExceptionStateForTests(): void {
|
||||
recentHandledPrimitives.length = 0;
|
||||
}
|
||||
|
|
@ -7,7 +7,6 @@ import {
|
|||
type CompletedCommandSpan,
|
||||
} from './command-hook.js';
|
||||
import { shutdownTelemetryEmitter, trackTelemetryEvent } from './emitter.js';
|
||||
import { reportException, type ExceptionContext } from './exception.js';
|
||||
import {
|
||||
buildCommonEnvelope,
|
||||
buildTelemetryEvent,
|
||||
|
|
@ -18,8 +17,8 @@ import {
|
|||
import { computeTelemetryProjectId, loadTelemetryIdentity } from './identity.js';
|
||||
import { buildProjectStackSnapshotFields } from './project-snapshot.js';
|
||||
|
||||
export { beginCommandSpan, completeCommandSpan, reportException, shutdownTelemetryEmitter };
|
||||
export type { CommandOutcome, CompletedCommandSpan, ExceptionContext };
|
||||
export { beginCommandSpan, completeCommandSpan, shutdownTelemetryEmitter };
|
||||
export type { CommandOutcome, CompletedCommandSpan };
|
||||
|
||||
export async function showTelemetryNoticeIfNeeded(io: KtxCliIo, packageInfo: KtxCliPackageInfo): Promise<void> {
|
||||
const identity = await loadTelemetryIdentity({
|
||||
|
|
|
|||
|
|
@ -1,117 +0,0 @@
|
|||
import { resolveKtxConfigReference } from '../context/core/config-reference.js';
|
||||
import { loadKtxProject, type KtxLocalProject } from '../context/project/project.js';
|
||||
|
||||
const SENSITIVE_KEY =
|
||||
/(password|secret|token|api[_-]?key|auth[_-]?token|auth_token_ref|private[_-]?key|passphrase|credential|authorization|url)$/i;
|
||||
|
||||
type TelemetryRedactionProject = Pick<KtxLocalProject, 'config' | 'projectDir'>;
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function addSecret(values: string[], value: string | undefined): void {
|
||||
const trimmed = value?.trim();
|
||||
if (trimmed && !values.includes(trimmed)) {
|
||||
values.push(trimmed);
|
||||
}
|
||||
}
|
||||
|
||||
function tryResolve(value: string, env: NodeJS.ProcessEnv): string | undefined {
|
||||
try {
|
||||
return resolveKtxConfigReference(value, env);
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function addUrlCredentials(values: string[], value: string): void {
|
||||
try {
|
||||
const parsed = new URL(value);
|
||||
addSecret(values, parsed.password ? decodeURIComponent(parsed.password) : undefined);
|
||||
addSecret(values, parsed.username ? decodeURIComponent(parsed.username) : undefined);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
function collectFromRecord(input: unknown, env: NodeJS.ProcessEnv, values: string[]): void {
|
||||
if (Array.isArray(input)) {
|
||||
for (const item of input) {
|
||||
collectFromRecord(item, env, values);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (!isRecord(input)) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (const [key, raw] of Object.entries(input)) {
|
||||
if (isRecord(raw) || Array.isArray(raw)) {
|
||||
collectFromRecord(raw, env, values);
|
||||
continue;
|
||||
}
|
||||
if (typeof raw !== 'string' || !SENSITIVE_KEY.test(key)) {
|
||||
continue;
|
||||
}
|
||||
const resolved = tryResolve(raw, env);
|
||||
addSecret(values, resolved);
|
||||
if (resolved) {
|
||||
addUrlCredentials(values, resolved);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function collectLlmSecrets(project: TelemetryRedactionProject, env: NodeJS.ProcessEnv, values: string[]): void {
|
||||
collectFromRecord(project.config.llm.provider, env, values);
|
||||
}
|
||||
|
||||
function collectEmbeddingSecrets(project: TelemetryRedactionProject, env: NodeJS.ProcessEnv, values: string[]): void {
|
||||
collectFromRecord(project.config.ingest.embeddings, env, values);
|
||||
collectFromRecord(project.config.scan.enrichment.embeddings, env, values);
|
||||
}
|
||||
|
||||
function collectConnectionSecrets(
|
||||
project: TelemetryRedactionProject,
|
||||
connectionId: string | undefined,
|
||||
env: NodeJS.ProcessEnv,
|
||||
values: string[],
|
||||
): void {
|
||||
if (!connectionId) {
|
||||
return;
|
||||
}
|
||||
collectFromRecord(project.config.connections[connectionId], env, values);
|
||||
}
|
||||
|
||||
export async function collectTelemetryRedactionSecrets(input: {
|
||||
project?: TelemetryRedactionProject;
|
||||
projectDir?: string;
|
||||
connectionId?: string;
|
||||
includeLlm?: boolean;
|
||||
includeEmbeddings?: boolean;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
}): Promise<string[]> {
|
||||
const env = input.env ?? process.env;
|
||||
let project = input.project;
|
||||
if (!project && input.projectDir) {
|
||||
try {
|
||||
project = await loadKtxProject({ projectDir: input.projectDir });
|
||||
} catch {
|
||||
project = undefined;
|
||||
}
|
||||
}
|
||||
if (!project) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const values: string[] = [];
|
||||
if (input.includeLlm) {
|
||||
collectLlmSecrets(project, env, values);
|
||||
}
|
||||
if (input.includeEmbeddings) {
|
||||
collectEmbeddingSecrets(project, env, values);
|
||||
}
|
||||
collectConnectionSecrets(project, input.connectionId, env, values);
|
||||
return values;
|
||||
}
|
||||
|
|
@ -1,45 +0,0 @@
|
|||
import { renameSync, writeFileSync } from 'node:fs';
|
||||
import { mkdir, readFile } from 'node:fs/promises';
|
||||
import { homedir } from 'node:os';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { z } from 'zod';
|
||||
|
||||
const updateCheckCacheSchema = z
|
||||
.object({
|
||||
checkedAt: z.string(),
|
||||
channel: z.enum(['latest', 'next']),
|
||||
installedVersion: z.string(),
|
||||
latestForChannel: z.string(),
|
||||
lastNoticeAt: z.string().optional(),
|
||||
})
|
||||
.strict();
|
||||
|
||||
export type UpdateCheckCache = z.infer<typeof updateCheckCacheSchema>;
|
||||
|
||||
/** @internal */
|
||||
export function updateCheckCachePath(homeDir = homedir()): string {
|
||||
return join(homeDir, '.ktx', 'update-check.json');
|
||||
}
|
||||
|
||||
export async function readUpdateCheckCache(options: { homeDir?: string } = {}): Promise<UpdateCheckCache | null> {
|
||||
try {
|
||||
return updateCheckCacheSchema.parse(JSON.parse(await readFile(updateCheckCachePath(options.homeDir), 'utf-8')));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function writeUpdateCheckCache(
|
||||
value: UpdateCheckCache,
|
||||
options: { homeDir?: string } = {},
|
||||
): Promise<void> {
|
||||
try {
|
||||
const path = updateCheckCachePath(options.homeDir);
|
||||
await mkdir(dirname(path), { recursive: true });
|
||||
const tempPath = `${path}.${process.pid}.${Date.now()}.tmp`;
|
||||
writeFileSync(tempPath, `${JSON.stringify(value, null, 2)}\n`, 'utf-8');
|
||||
renameSync(tempPath, path);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,43 +0,0 @@
|
|||
import semver from 'semver';
|
||||
|
||||
export type UpdateChannel = 'latest' | 'next';
|
||||
|
||||
export type UpdateDecision =
|
||||
| { status: 'skip' }
|
||||
| { status: 'upToDate'; channel: UpdateChannel; target: string }
|
||||
| { status: 'available'; channel: UpdateChannel; target: string };
|
||||
|
||||
/** @internal */
|
||||
export function inferUpdateChannel(installed: string): UpdateChannel | null {
|
||||
const parsed = semver.parse(installed);
|
||||
if (!parsed || installed === '0.0.0') {
|
||||
return null;
|
||||
}
|
||||
|
||||
const [prereleaseId] = parsed.prerelease;
|
||||
if (prereleaseId === undefined) {
|
||||
return 'latest';
|
||||
}
|
||||
if (prereleaseId === 'rc') {
|
||||
return 'next';
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function decideUpdate(installed: string, distTags: Record<string, string>): UpdateDecision {
|
||||
const channel = inferUpdateChannel(installed);
|
||||
if (!channel || !semver.valid(installed)) {
|
||||
return { status: 'skip' };
|
||||
}
|
||||
|
||||
const target = distTags[channel];
|
||||
if (!target || !semver.valid(target)) {
|
||||
return { status: 'skip' };
|
||||
}
|
||||
|
||||
if (semver.gt(target, installed)) {
|
||||
return { status: 'available', channel, target };
|
||||
}
|
||||
|
||||
return { status: 'upToDate', channel, target };
|
||||
}
|
||||
|
|
@ -1,52 +0,0 @@
|
|||
import { request as httpsRequest } from 'node:https';
|
||||
import { URL } from 'node:url';
|
||||
import { z } from 'zod';
|
||||
|
||||
const DIST_TAGS_URL = new URL('https://registry.npmjs.org/-/package/@kaelio/ktx/dist-tags');
|
||||
const distTagsSchema = z.record(z.string(), z.string());
|
||||
|
||||
function parseDistTags(raw: string): Record<string, string> {
|
||||
return distTagsSchema.parse(JSON.parse(raw));
|
||||
}
|
||||
|
||||
export function fetchDistTags(): Promise<Record<string, string>> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const request = httpsRequest(
|
||||
DIST_TAGS_URL,
|
||||
{
|
||||
method: 'GET',
|
||||
headers: {
|
||||
accept: 'application/json',
|
||||
},
|
||||
},
|
||||
(response) => {
|
||||
const chunks: Buffer[] = [];
|
||||
response.on('data', (chunk: Buffer | string) => {
|
||||
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
||||
});
|
||||
response.on('end', () => {
|
||||
const text = Buffer.concat(chunks).toString('utf8');
|
||||
const statusCode = response.statusCode ?? 0;
|
||||
if (statusCode < 200 || statusCode >= 300) {
|
||||
reject(new Error(`npm dist-tags request failed with ${statusCode}: ${text}`));
|
||||
return;
|
||||
}
|
||||
try {
|
||||
resolve(parseDistTags(text));
|
||||
} catch (error) {
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
request.on('socket', (socket) => {
|
||||
socket.unref();
|
||||
});
|
||||
request.on('error', reject);
|
||||
request.setTimeout(5000, () => {
|
||||
request.destroy(new Error('npm dist-tags request timed out'));
|
||||
});
|
||||
request.end();
|
||||
});
|
||||
}
|
||||
|
|
@ -1,187 +0,0 @@
|
|||
import type { KtxCliIo } from '../cli-runtime.js';
|
||||
import { cyan, dim, type CliStyleEnv } from '../clack.js';
|
||||
import { resolveOutputMode } from '../io/mode.js';
|
||||
import { type UpdateCheckCache, readUpdateCheckCache, writeUpdateCheckCache } from './cache.js';
|
||||
import { decideUpdate, inferUpdateChannel, type UpdateChannel } from './channel.js';
|
||||
import { fetchDistTags as defaultFetchDistTags } from './registry.js';
|
||||
|
||||
const DAY_MS = 24 * 60 * 60 * 1000;
|
||||
|
||||
/** @internal */
|
||||
export interface UpdateCheckEnv extends NodeJS.ProcessEnv, CliStyleEnv {
|
||||
CI?: string;
|
||||
DO_NOT_TRACK?: string;
|
||||
KTX_NO_UPDATE_CHECK?: string;
|
||||
KTX_OUTPUT?: string;
|
||||
NO_UPDATE_NOTIFIER?: string;
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export interface UpdateCheckCommandOptions {
|
||||
format?: unknown;
|
||||
json?: unknown;
|
||||
output?: unknown;
|
||||
}
|
||||
|
||||
export interface PrepareUpdateCheckNoticeOptions {
|
||||
commandOptions?: UpdateCheckCommandOptions;
|
||||
env?: UpdateCheckEnv;
|
||||
fetchDistTags?: () => Promise<Record<string, string>>;
|
||||
homeDir?: string;
|
||||
installedVersion: string;
|
||||
io: KtxCliIo;
|
||||
now?: () => Date;
|
||||
}
|
||||
|
||||
export interface PreparedUpdateCheckNotice {
|
||||
notice: string | null;
|
||||
}
|
||||
|
||||
function truthy(value: string | undefined): boolean {
|
||||
return value !== undefined && value !== '' && value !== '0' && value !== 'false';
|
||||
}
|
||||
|
||||
function commandRequestsJson(options: UpdateCheckCommandOptions | undefined): boolean {
|
||||
return options?.json === true || options?.output === 'json' || options?.format === 'json';
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function shouldSuppressUpdateCheck(args: {
|
||||
commandOptions?: UpdateCheckCommandOptions;
|
||||
env?: UpdateCheckEnv;
|
||||
io: KtxCliIo;
|
||||
}): boolean {
|
||||
const env = args.env ?? process.env;
|
||||
if (truthy(env.KTX_NO_UPDATE_CHECK) || truthy(env.NO_UPDATE_NOTIFIER) || truthy(env.DO_NOT_TRACK)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (commandRequestsJson(args.commandOptions) || truthy(env.CI) || args.io.stdout.isTTY !== true) {
|
||||
return true;
|
||||
}
|
||||
|
||||
try {
|
||||
const mode = resolveOutputMode({
|
||||
json: false,
|
||||
io: args.io,
|
||||
env,
|
||||
});
|
||||
return mode !== 'pretty';
|
||||
} catch {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function renderUpdateNotice(args: {
|
||||
channel: UpdateChannel;
|
||||
env?: CliStyleEnv;
|
||||
installedVersion: string;
|
||||
targetVersion: string;
|
||||
}): string {
|
||||
const command = args.channel === 'next' ? 'npm i -g @kaelio/ktx@next' : 'npm i -g @kaelio/ktx';
|
||||
return `${cyan('↑', args.env)} Update available: ktx ${args.installedVersion} → ${args.targetVersion}\n ${dim(command, args.env)}\n`;
|
||||
}
|
||||
|
||||
function timestampMs(value: string | undefined): number | null {
|
||||
if (!value) {
|
||||
return null;
|
||||
}
|
||||
const parsed = Date.parse(value);
|
||||
return Number.isNaN(parsed) ? null : parsed;
|
||||
}
|
||||
|
||||
function elapsedAtLeast(value: string | undefined, now: Date, intervalMs: number): boolean {
|
||||
const previous = timestampMs(value);
|
||||
if (previous === null) {
|
||||
return true;
|
||||
}
|
||||
return now.getTime() - previous >= intervalMs;
|
||||
}
|
||||
|
||||
function shouldRefreshCache(cache: UpdateCheckCache | null, installedVersion: string, now: Date): boolean {
|
||||
if (!cache || cache.installedVersion !== installedVersion) {
|
||||
return true;
|
||||
}
|
||||
return elapsedAtLeast(cache.checkedAt, now, DAY_MS);
|
||||
}
|
||||
|
||||
async function refreshUpdateCache(args: {
|
||||
cache: UpdateCheckCache | null;
|
||||
fetchDistTags: () => Promise<Record<string, string>>;
|
||||
homeDir?: string;
|
||||
installedVersion: string;
|
||||
now: Date;
|
||||
}): Promise<void> {
|
||||
const distTags = await args.fetchDistTags();
|
||||
const decision = decideUpdate(args.installedVersion, distTags);
|
||||
if (decision.status === 'skip') {
|
||||
return;
|
||||
}
|
||||
|
||||
await writeUpdateCheckCache(
|
||||
{
|
||||
checkedAt: args.now.toISOString(),
|
||||
channel: decision.channel,
|
||||
installedVersion: args.installedVersion,
|
||||
latestForChannel: decision.target,
|
||||
...(args.cache?.installedVersion === args.installedVersion && args.cache.channel === decision.channel
|
||||
? { lastNoticeAt: args.cache.lastNoticeAt }
|
||||
: {}),
|
||||
},
|
||||
{ homeDir: args.homeDir },
|
||||
);
|
||||
}
|
||||
|
||||
export async function prepareUpdateCheckNotice(
|
||||
options: PrepareUpdateCheckNoticeOptions,
|
||||
): Promise<PreparedUpdateCheckNotice> {
|
||||
const env = options.env ?? process.env;
|
||||
const now = (options.now ?? (() => new Date()))();
|
||||
const fetchDistTags = options.fetchDistTags ?? defaultFetchDistTags;
|
||||
|
||||
if (
|
||||
shouldSuppressUpdateCheck({
|
||||
commandOptions: options.commandOptions,
|
||||
env,
|
||||
io: options.io,
|
||||
})
|
||||
) {
|
||||
return { notice: null };
|
||||
}
|
||||
|
||||
if (!inferUpdateChannel(options.installedVersion)) {
|
||||
return { notice: null };
|
||||
}
|
||||
|
||||
let cache = await readUpdateCheckCache({ homeDir: options.homeDir });
|
||||
let notice: string | null = null;
|
||||
|
||||
if (cache?.installedVersion === options.installedVersion) {
|
||||
const decision = decideUpdate(options.installedVersion, {
|
||||
[cache.channel]: cache.latestForChannel,
|
||||
});
|
||||
if (decision.status === 'available' && elapsedAtLeast(cache.lastNoticeAt, now, DAY_MS)) {
|
||||
notice = renderUpdateNotice({
|
||||
channel: decision.channel,
|
||||
env,
|
||||
installedVersion: options.installedVersion,
|
||||
targetVersion: decision.target,
|
||||
});
|
||||
cache = { ...cache, lastNoticeAt: now.toISOString() };
|
||||
await writeUpdateCheckCache(cache, { homeDir: options.homeDir });
|
||||
}
|
||||
}
|
||||
|
||||
if (shouldRefreshCache(cache, options.installedVersion, now)) {
|
||||
void refreshUpdateCache({
|
||||
cache,
|
||||
fetchDistTags,
|
||||
homeDir: options.homeDir,
|
||||
installedVersion: options.installedVersion,
|
||||
now,
|
||||
}).catch(() => {});
|
||||
}
|
||||
|
||||
return { notice };
|
||||
}
|
||||
|
|
@ -7,12 +7,6 @@ import { runCommanderKtxCli } from '../src/cli-program.js';
|
|||
import type { KtxCliDeps, KtxCliIo, KtxCliPackageInfo } from '../src/cli-runtime.js';
|
||||
import { TELEMETRY_NOTICE } from '../src/telemetry/identity.js';
|
||||
|
||||
const reportExceptionMock = vi.hoisted(() => vi.fn(async () => {}));
|
||||
|
||||
vi.mock('../src/telemetry/exception.js', () => ({
|
||||
reportException: reportExceptionMock,
|
||||
}));
|
||||
|
||||
function makeIo(stdoutIsTTY = true): { io: KtxCliIo; stdout: () => string; stderr: () => string } {
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
|
|
@ -49,7 +43,6 @@ describe('runCommanderKtxCli telemetry', () => {
|
|||
vi.stubEnv('CI', '');
|
||||
vi.stubEnv('KTX_TELEMETRY_DISABLED', '');
|
||||
vi.stubEnv('DO_NOT_TRACK', '');
|
||||
reportExceptionMock.mockClear();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
|
|
@ -138,30 +131,4 @@ describe('runCommanderKtxCli telemetry', () => {
|
|||
await expect(runCommanderKtxCli(['unknown'], unknownIo.io, {}, info, { runInit: async () => 0 })).resolves.toBe(1);
|
||||
expect(unknownIo.stderr()).not.toContain('[telemetry]');
|
||||
});
|
||||
|
||||
it('reports genuine top-level command catches as handled exceptions', async () => {
|
||||
const io = makeIo(true);
|
||||
const deps: KtxCliDeps = {
|
||||
doctor: async () => {
|
||||
throw new Error('status failed');
|
||||
},
|
||||
};
|
||||
|
||||
await expect(
|
||||
runCommanderKtxCli(
|
||||
['--project-dir', tempDir, 'status', '--json'],
|
||||
io.io,
|
||||
deps,
|
||||
info,
|
||||
{ runInit: async () => 0 },
|
||||
),
|
||||
).resolves.toBe(1);
|
||||
|
||||
expect(reportExceptionMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
context: expect.objectContaining({ source: 'ktx status', handled: true, fatal: false }),
|
||||
projectDir: tempDir,
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -10,12 +10,6 @@ import type { KtxConnectionDriver, KtxScanConnector } from '../src/context/scan/
|
|||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { runKtxConnection } from '../src/connection.js';
|
||||
|
||||
const reportExceptionMock = vi.hoisted(() => vi.fn(async () => {}));
|
||||
|
||||
vi.mock('../src/telemetry/exception.js', () => ({
|
||||
reportException: reportExceptionMock,
|
||||
}));
|
||||
|
||||
function stripAnsi(s: string): string {
|
||||
return s.replace(/\[[0-9;]*m/g, '');
|
||||
}
|
||||
|
|
@ -44,7 +38,7 @@ function makeIo() {
|
|||
|
||||
function nativeConnector(
|
||||
driver: KtxConnectionDriver,
|
||||
testResult: { success: true } | { success: false; error: string; cause?: unknown } = { success: true },
|
||||
testResult: { success: true } | { success: false; error: string } = { success: true },
|
||||
) {
|
||||
const testConnection = vi.fn(async () => testResult);
|
||||
const cleanup = vi.fn(async () => undefined);
|
||||
|
|
@ -78,7 +72,6 @@ describe('runKtxConnection', () => {
|
|||
|
||||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'ktx-cli-connection-'));
|
||||
reportExceptionMock.mockClear();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
|
|
@ -172,13 +165,12 @@ describe('runKtxConnection', () => {
|
|||
it('records the raw errorDetail in connection_test telemetry when a native test fails', async () => {
|
||||
vi.stubEnv('KTX_TELEMETRY_DEBUG', '1');
|
||||
vi.stubEnv('CI', '');
|
||||
vi.stubEnv('DATABASE_URL', 'postgres://svc:db-url-password@db.example.test/analytics'); // pragma: allowlist secret
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await initKtxProject({ projectDir });
|
||||
await writeConnections(projectDir, {
|
||||
warehouse: { driver: 'postgres', url: 'env:DATABASE_URL' },
|
||||
warehouse: { driver: 'sqlite' },
|
||||
});
|
||||
const { connector } = nativeConnector('postgres', { success: false, error: 'database file is unreadable' });
|
||||
const { connector } = nativeConnector('sqlite', { success: false, error: 'database file is unreadable' });
|
||||
const io = makeIo();
|
||||
|
||||
const code = await runKtxConnection({ command: 'test', projectDir, connectionId: 'warehouse' }, io.io, {
|
||||
|
|
@ -189,44 +181,6 @@ describe('runKtxConnection', () => {
|
|||
expect(io.stderr()).toContain('"event":"connection_test"');
|
||||
expect(io.stderr()).toContain('"outcome":"error"');
|
||||
expect(io.stderr()).toContain('"errorDetail":"database file is unreadable"');
|
||||
expect(reportExceptionMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
context: expect.objectContaining({ source: 'connection test', handled: true, fatal: false }),
|
||||
projectDir,
|
||||
redactionSecrets: expect.arrayContaining([
|
||||
'postgres://svc:db-url-password@db.example.test/analytics', // pragma: allowlist secret
|
||||
'db-url-password',
|
||||
]),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('preserves the driver error class and code in connection_test telemetry', async () => {
|
||||
vi.stubEnv('KTX_TELEMETRY_DEBUG', '1');
|
||||
vi.stubEnv('CI', '');
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await initKtxProject({ projectDir });
|
||||
await writeConnections(projectDir, {
|
||||
warehouse: { driver: 'sqlserver', host: 'db.example.test', database: 'analytics', username: 'svc_ro' },
|
||||
});
|
||||
class ConnectionError extends Error {
|
||||
readonly code = 'ELOGIN';
|
||||
}
|
||||
const driverError = new ConnectionError("Login failed for user 'svc_ro'.");
|
||||
const { connector } = nativeConnector('sqlserver', {
|
||||
success: false,
|
||||
error: driverError.message,
|
||||
cause: driverError,
|
||||
});
|
||||
const io = makeIo();
|
||||
|
||||
const code = await runKtxConnection({ command: 'test', projectDir, connectionId: 'warehouse' }, io.io, {
|
||||
createScanConnector: vi.fn(async () => connector),
|
||||
});
|
||||
|
||||
expect(code).toBe(1);
|
||||
expect(io.stderr()).toContain('"errorClass":"ConnectionError"');
|
||||
expect(io.stderr()).toContain('"errorDetail":"ELOGIN: Login failed for user \'svc_ro\'."');
|
||||
});
|
||||
|
||||
it('reports the connector error and still cleans up when native testConnection fails', async () => {
|
||||
|
|
|
|||
|
|
@ -1,31 +0,0 @@
|
|||
import { describe, expect, it, vi } from 'vitest';
|
||||
import { createAbortError, isAbortError, linkAbortSignal, throwIfAborted } from '../../../src/context/core/abort.js';
|
||||
|
||||
describe('abort helpers', () => {
|
||||
it('recognizes DOMException abort errors and common abort-shaped errors', () => {
|
||||
expect(isAbortError(createAbortError())).toBe(true);
|
||||
expect(isAbortError(Object.assign(new Error('cancelled'), { name: 'AbortError' }))).toBe(true);
|
||||
expect(isAbortError(Object.assign(new Error('operation aborted'), { code: 'ABORT_ERR' }))).toBe(true);
|
||||
expect(isAbortError(new Error('ordinary failure'))).toBe(false);
|
||||
});
|
||||
|
||||
it('throws when the provided signal is already aborted', () => {
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
|
||||
expect(() => throwIfAborted(controller.signal)).toThrow(/Aborted/);
|
||||
});
|
||||
|
||||
it('links a child controller to a parent signal and removes the listener on dispose', () => {
|
||||
const parent = new AbortController();
|
||||
const child = linkAbortSignal(parent.signal);
|
||||
|
||||
expect(child.controller.signal.aborted).toBe(false);
|
||||
parent.abort();
|
||||
expect(child.controller.signal.aborted).toBe(true);
|
||||
|
||||
const removeSpy = vi.spyOn(parent.signal, 'removeEventListener');
|
||||
child.dispose();
|
||||
expect(removeSpy).toHaveBeenCalledWith('abort', expect.any(Function));
|
||||
});
|
||||
});
|
||||
|
|
@ -64,27 +64,6 @@ function sqlAnalysis(tablesById: Record<string, Array<{ catalog: string | null;
|
|||
};
|
||||
}
|
||||
|
||||
function sqlAnalysisWithErrors(
|
||||
tablesById: Record<string, Array<{ catalog: string | null; db: string | null; name: string }>>,
|
||||
errorIds: string[],
|
||||
): SqlAnalysisPort {
|
||||
const errors = new Set(errorIds);
|
||||
return {
|
||||
analyzeForFingerprint: vi.fn(),
|
||||
analyzeBatch: vi.fn(async (items: SqlAnalysisBatchItem[]): Promise<Map<string, SqlAnalysisBatchResult>> =>
|
||||
new Map<string, SqlAnalysisBatchResult>(
|
||||
items.map((item) => [
|
||||
item.id,
|
||||
errors.has(item.id)
|
||||
? { tablesTouched: [], columnsByClause: {}, error: 'parse boom' }
|
||||
: { tablesTouched: tablesById[item.id] ?? [], columnsByClause: {} },
|
||||
]),
|
||||
),
|
||||
),
|
||||
validateReadOnly: vi.fn(async () => ({ ok: true })),
|
||||
};
|
||||
}
|
||||
|
||||
function llm(decisions: Array<{ role: string; exclude: boolean; reason: string }>): KtxLlmRuntimePort {
|
||||
const generateObject = vi.fn(async () => ({ roles: decisions })) as KtxLlmRuntimePort['generateObject'];
|
||||
return {
|
||||
|
|
@ -219,7 +198,6 @@ describe('query-history filter picker', () => {
|
|||
consideredRoleCount: 0,
|
||||
skipped: { reason: 'no-llm' },
|
||||
warnings: [],
|
||||
parseFailedTemplateIds: [],
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -249,32 +227,6 @@ describe('query-history filter picker', () => {
|
|||
expect(proposal.skipped).toEqual({ reason: 'no-in-scope-history' });
|
||||
});
|
||||
|
||||
it('records parse failures as template ids, not warnings', async () => {
|
||||
const proposal = await proposeQueryHistoryServiceAccountFilters({
|
||||
connectionId: 'warehouse',
|
||||
dialect: 'postgres',
|
||||
queryClient: {},
|
||||
reader: reader(
|
||||
aggregate({
|
||||
templateId: 'good',
|
||||
canonicalSql: 'select * from analytics.orders',
|
||||
topUsers: [{ user: 'analyst', executions: 30 }],
|
||||
}),
|
||||
aggregate({
|
||||
templateId: 'broken',
|
||||
canonicalSql: 'select * from where',
|
||||
topUsers: [{ user: 'analyst', executions: 5 }],
|
||||
}),
|
||||
),
|
||||
sqlAnalysis: sqlAnalysisWithErrors({ good: [{ catalog: null, db: 'analytics', name: 'orders' }] }, ['broken']),
|
||||
llmRuntime: llm([]),
|
||||
pullConfig: { dialect: 'postgres', enabledSchemas: ['analytics'], filters: { dropTrivialProbes: true } },
|
||||
});
|
||||
|
||||
expect(proposal.parseFailedTemplateIds).toEqual(['broken']);
|
||||
expect(proposal.warnings).toEqual([]);
|
||||
});
|
||||
|
||||
it('keeps clean in-scope history when the model excludes nothing', async () => {
|
||||
const proposal = await proposeQueryHistoryServiceAccountFilters({
|
||||
connectionId: 'warehouse',
|
||||
|
|
|
|||
|
|
@ -426,177 +426,6 @@ describe('IngestBundleRunner — Stages 1 → 7', () => {
|
|||
);
|
||||
});
|
||||
|
||||
it('uses the rate-limit governor for work-unit start slots', async () => {
|
||||
const deps = makeDeps();
|
||||
const acquireWorkSlot = vi.fn(async () => vi.fn());
|
||||
const runner = buildRunner(deps, {
|
||||
settings: {
|
||||
probeRowCount: 1,
|
||||
memoryIngestionModel: 'test-model',
|
||||
workUnitMaxConcurrency: 2,
|
||||
rateLimitGovernor: { acquireWorkSlot, subscribe: vi.fn(() => vi.fn()) } as never,
|
||||
},
|
||||
});
|
||||
deps.adapter.chunk.mockResolvedValue({
|
||||
workUnits: [
|
||||
{ unitKey: 'u1', rawFiles: ['a.yml'], peerFileIndex: [], dependencyPaths: [] },
|
||||
{ unitKey: 'u2', rawFiles: ['b.yml'], peerFileIndex: [], dependencyPaths: [] },
|
||||
],
|
||||
});
|
||||
(runner as any).stageRawFilesStage1 = vi.fn().mockResolvedValue({
|
||||
currentHashes: new Map([
|
||||
['a.yml', 'h1'],
|
||||
['b.yml', 'h2'],
|
||||
]),
|
||||
rawDirInWorktree: 'raw-sources/c1/fake/s',
|
||||
});
|
||||
(runner as any).resolveStagedDir = vi.fn().mockResolvedValue('/tmp/stage/upload-x');
|
||||
|
||||
await runner.run({
|
||||
jobId: 'j1',
|
||||
connectionId: 'c1',
|
||||
sourceKey: 'fake',
|
||||
trigger: 'upload',
|
||||
bundleRef: { kind: 'upload', uploadId: 'upload-x' },
|
||||
});
|
||||
|
||||
expect(acquireWorkSlot).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('passes the job abort signal into rate-limit work-unit slots', async () => {
|
||||
const deps = makeDeps();
|
||||
const controller = new AbortController();
|
||||
const acquireWorkSlot = vi.fn(async () => vi.fn());
|
||||
const runner = buildRunner(deps, {
|
||||
settings: {
|
||||
probeRowCount: 1,
|
||||
memoryIngestionModel: 'test-model',
|
||||
workUnitMaxConcurrency: 1,
|
||||
rateLimitGovernor: { acquireWorkSlot, subscribe: vi.fn(() => vi.fn()) } as never,
|
||||
},
|
||||
});
|
||||
(runner as any).stageRawFilesStage1 = vi.fn().mockResolvedValue({
|
||||
currentHashes: new Map([['a.yml', 'h1']]),
|
||||
rawDirInWorktree: 'raw-sources/c1/fake/s',
|
||||
});
|
||||
(runner as any).resolveStagedDir = vi.fn().mockResolvedValue('/tmp/stage/upload-x');
|
||||
|
||||
await runner.run(
|
||||
{
|
||||
jobId: 'j1',
|
||||
connectionId: 'c1',
|
||||
sourceKey: 'fake',
|
||||
trigger: 'upload',
|
||||
bundleRef: { kind: 'upload', uploadId: 'upload-x' },
|
||||
},
|
||||
{ jobId: 'j1', abortSignal: controller.signal, startPhase: () => new TestJobContext('j1', null, async () => undefined, async () => undefined) } as any,
|
||||
);
|
||||
|
||||
expect(acquireWorkSlot).toHaveBeenCalledWith(controller.signal);
|
||||
});
|
||||
|
||||
it('does not convert aborted work-unit agent loops into failed work units', async () => {
|
||||
const deps = makeDeps();
|
||||
const controller = new AbortController();
|
||||
deps.agentRunner.runLoop.mockImplementation(async () => {
|
||||
controller.abort();
|
||||
throw new DOMException('Aborted', 'AbortError');
|
||||
});
|
||||
const runner = buildRunner(deps, {
|
||||
settings: {
|
||||
probeRowCount: 1,
|
||||
memoryIngestionModel: 'test-model',
|
||||
workUnitMaxConcurrency: 1,
|
||||
},
|
||||
});
|
||||
(runner as any).stageRawFilesStage1 = vi.fn().mockResolvedValue({
|
||||
currentHashes: new Map([['a.yml', 'h1']]),
|
||||
rawDirInWorktree: 'raw-sources/c1/fake/s',
|
||||
});
|
||||
(runner as any).resolveStagedDir = vi.fn().mockResolvedValue('/tmp/stage/upload-x');
|
||||
|
||||
await expect(
|
||||
runner.run(
|
||||
{
|
||||
jobId: 'j1',
|
||||
connectionId: 'c1',
|
||||
sourceKey: 'fake',
|
||||
trigger: 'upload',
|
||||
bundleRef: { kind: 'upload', uploadId: 'upload-x' },
|
||||
},
|
||||
{ jobId: 'j1', abortSignal: controller.signal, startPhase: () => new TestJobContext('j1', null, async () => undefined, async () => undefined) } as any,
|
||||
),
|
||||
).rejects.toThrow(/Aborted/);
|
||||
|
||||
expect(deps.runsRepo.markFailed).toHaveBeenCalledWith('run-1');
|
||||
expect(deps.reportsRepo.create).not.toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
body: expect.objectContaining({
|
||||
failedWorkUnits: expect.arrayContaining(['u1']),
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('emits trace and memory-flow status for rate-limit waits', async () => {
|
||||
const deps = makeDeps();
|
||||
let subscriber: ((state: any) => void) | undefined;
|
||||
const memoryFlow = createMemoryFlowLiveBuffer(bundleReplayInput());
|
||||
const runner = buildRunner(deps, {
|
||||
settings: {
|
||||
probeRowCount: 1,
|
||||
memoryIngestionModel: 'test-model',
|
||||
rateLimitGovernor: {
|
||||
acquireWorkSlot: vi.fn(async () => vi.fn()),
|
||||
subscribe: vi.fn((cb: (state: any) => void) => {
|
||||
subscriber = cb;
|
||||
return vi.fn();
|
||||
}),
|
||||
} as never,
|
||||
},
|
||||
});
|
||||
(runner as any).runInner = async (_job: any, ctx: any) => {
|
||||
subscriber?.({
|
||||
kind: 'wait_tick',
|
||||
provider: 'claude-subscription',
|
||||
rateLimitType: 'five_hour',
|
||||
resumeAtMs: 2_000,
|
||||
remainingMs: 1_000,
|
||||
});
|
||||
ctx.memoryFlow.emit({ type: 'report_created', runId: 'run-1' });
|
||||
return {
|
||||
runId: 'run-1',
|
||||
syncId: 'sync-1',
|
||||
diffSummary: { added: 0, modified: 0, deleted: 0, unchanged: 0 },
|
||||
workUnitCount: 0,
|
||||
failedWorkUnits: [],
|
||||
artifactsWritten: 0,
|
||||
commitSha: null,
|
||||
};
|
||||
};
|
||||
|
||||
await runner.run(
|
||||
{
|
||||
jobId: 'j1',
|
||||
connectionId: 'c1',
|
||||
sourceKey: 'fake',
|
||||
trigger: 'upload',
|
||||
bundleRef: { kind: 'upload', uploadId: 'upload-x' },
|
||||
},
|
||||
{ memoryFlow } as any,
|
||||
);
|
||||
|
||||
expect(memoryFlow.snapshot().events).toContainEqual(
|
||||
expect.objectContaining({
|
||||
type: 'rate_limit_wait',
|
||||
provider: 'claude-subscription',
|
||||
rateLimitType: 'five_hour',
|
||||
resumeAtMs: 2_000,
|
||||
remainingMs: 1_000,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('fails before squash when reconciliation leaves a touched wiki page with dangling refs', async () => {
|
||||
const deps = makeDeps();
|
||||
let currentToolSession: any = null;
|
||||
|
|
|
|||
|
|
@ -301,7 +301,6 @@ describe('createLocalBundleIngestRuntime', () => {
|
|||
'memoryIngestionModel',
|
||||
'probeRowCount',
|
||||
'profileIngest',
|
||||
'rateLimitGovernor',
|
||||
'workUnitFailureMode',
|
||||
'workUnitMaxConcurrency',
|
||||
'workUnitStepBudget',
|
||||
|
|
|
|||
|
|
@ -146,29 +146,6 @@ describe('memory-flow schemas', () => {
|
|||
expect(parsed.events).toContainEqual({ type: 'stage_skipped', stage: 'actions', reason: 'requires LLM' });
|
||||
});
|
||||
|
||||
it('accepts rate-limit wait replay events', () => {
|
||||
expect(
|
||||
memoryFlowReplayInputSchema.parse({
|
||||
...snapshot(),
|
||||
events: [
|
||||
{
|
||||
type: 'rate_limit_wait',
|
||||
provider: 'claude-subscription',
|
||||
rateLimitType: 'five_hour',
|
||||
resumeAtMs: 2_000,
|
||||
remainingMs: 1_000,
|
||||
},
|
||||
],
|
||||
}).events[0],
|
||||
).toEqual({
|
||||
type: 'rate_limit_wait',
|
||||
provider: 'claude-subscription',
|
||||
rateLimitType: 'five_hour',
|
||||
resumeAtMs: 2_000,
|
||||
remainingMs: 1_000,
|
||||
});
|
||||
});
|
||||
|
||||
it('parses snapshot and closed stream events', () => {
|
||||
expect(memoryFlowStreamEventSchema.parse({ type: 'snapshot', snapshot: snapshot({ status: 'done' }) })).toEqual({
|
||||
type: 'snapshot',
|
||||
|
|
|
|||
|
|
@ -107,199 +107,6 @@ describe('AiSdkKtxLlmRuntime.runAgentLoop', () => {
|
|||
expect(result.error).toBe(err);
|
||||
});
|
||||
|
||||
it('reports AI SDK retry-after rate limits and retries through the governor', async () => {
|
||||
const waitForReady = vi.fn().mockResolvedValue(undefined);
|
||||
const report = vi.fn();
|
||||
const rateLimitError = Object.assign(new Error('too many requests'), {
|
||||
name: 'TooManyRequestsError',
|
||||
retryAfter: 2,
|
||||
statusCode: 429,
|
||||
});
|
||||
(generateText as any).mockRejectedValueOnce(rateLimitError).mockResolvedValueOnce({
|
||||
text: 'done',
|
||||
toolCalls: [],
|
||||
steps: [],
|
||||
usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 },
|
||||
});
|
||||
const runtime = new AiSdkKtxLlmRuntime({
|
||||
llmProvider: llmProvider as any,
|
||||
rateLimitGovernor: { waitForReady, report, maxRetryAttempts: () => 6 } as never,
|
||||
});
|
||||
|
||||
const result = await runtime.runAgentLoop({
|
||||
modelRole: 'candidateExtraction',
|
||||
systemPrompt: '',
|
||||
userPrompt: '',
|
||||
toolSet: {},
|
||||
stepBudget: 10,
|
||||
telemetryTags: {},
|
||||
});
|
||||
|
||||
expect(result.stopReason).toBe('natural');
|
||||
expect(report).toHaveBeenCalledWith({
|
||||
provider: 'anthropic-api',
|
||||
status: 'rejected',
|
||||
retryAfterMs: 2_000,
|
||||
rateLimitType: 'http_429',
|
||||
});
|
||||
expect(waitForReady).toHaveBeenCalledTimes(2);
|
||||
expect(generateText).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('does not retry AI SDK rate limits without a governor', async () => {
|
||||
const rateLimitError = Object.assign(new Error('too many requests'), {
|
||||
name: 'TooManyRequestsError',
|
||||
statusCode: 429,
|
||||
});
|
||||
(generateText as any).mockRejectedValue(rateLimitError);
|
||||
// The beforeEach runtime is constructed without a rateLimitGovernor.
|
||||
|
||||
const result = await runtime.runAgentLoop({
|
||||
modelRole: 'candidateExtraction',
|
||||
systemPrompt: '',
|
||||
userPrompt: '',
|
||||
toolSet: {},
|
||||
stepBudget: 10,
|
||||
telemetryTags: {},
|
||||
});
|
||||
|
||||
expect(result.stopReason).toBe('error');
|
||||
expect(generateText).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('honors a governor retry budget of one attempt without retrying', async () => {
|
||||
const waitForReady = vi.fn().mockResolvedValue(undefined);
|
||||
const report = vi.fn();
|
||||
const rateLimitError = Object.assign(new Error('too many requests'), {
|
||||
name: 'TooManyRequestsError',
|
||||
statusCode: 429,
|
||||
});
|
||||
(generateText as any).mockRejectedValue(rateLimitError);
|
||||
const runtime = new AiSdkKtxLlmRuntime({
|
||||
llmProvider: llmProvider as any,
|
||||
rateLimitGovernor: { waitForReady, report, maxRetryAttempts: () => 1 } as never,
|
||||
});
|
||||
|
||||
const result = await runtime.runAgentLoop({
|
||||
modelRole: 'candidateExtraction',
|
||||
systemPrompt: '',
|
||||
userPrompt: '',
|
||||
toolSet: {},
|
||||
stepBudget: 10,
|
||||
telemetryTags: {},
|
||||
});
|
||||
|
||||
expect(result.stopReason).toBe('error');
|
||||
expect(generateText).toHaveBeenCalledTimes(1);
|
||||
expect(report).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('reports Anthropic API response-header utilization to the governor', async () => {
|
||||
const waitForReady = vi.fn().mockResolvedValue(undefined);
|
||||
const report = vi.fn();
|
||||
(generateText as any).mockResolvedValue({
|
||||
text: 'done',
|
||||
toolCalls: [],
|
||||
steps: [],
|
||||
response: {
|
||||
headers: {
|
||||
'anthropic-ratelimit-requests-limit': '100',
|
||||
'anthropic-ratelimit-requests-remaining': '8',
|
||||
'anthropic-ratelimit-input-tokens-limit': '10000',
|
||||
'anthropic-ratelimit-input-tokens-remaining': '9000',
|
||||
},
|
||||
},
|
||||
});
|
||||
const runtime = new AiSdkKtxLlmRuntime({
|
||||
llmProvider: llmProvider as any,
|
||||
rateLimitGovernor: { waitForReady, report, maxRetryAttempts: () => 6 } as never,
|
||||
});
|
||||
|
||||
const result = await runtime.runAgentLoop({
|
||||
modelRole: 'candidateExtraction',
|
||||
systemPrompt: '',
|
||||
userPrompt: '',
|
||||
toolSet: {},
|
||||
stepBudget: 10,
|
||||
telemetryTags: {},
|
||||
});
|
||||
|
||||
expect(result.stopReason).toBe('natural');
|
||||
expect(report).toHaveBeenCalledWith({
|
||||
provider: 'anthropic-api',
|
||||
status: 'allowed',
|
||||
rateLimitType: 'rpm',
|
||||
utilization: 0.92,
|
||||
});
|
||||
});
|
||||
|
||||
it('reports generic x-ratelimit response-header utilization for Vertex providers', async () => {
|
||||
const waitForReady = vi.fn().mockResolvedValue(undefined);
|
||||
const report = vi.fn();
|
||||
const vertexProvider = {
|
||||
...llmProvider,
|
||||
getModel: vi.fn().mockReturnValue({ modelId: 'gemini-3-pro', provider: 'google-vertex' }),
|
||||
};
|
||||
(generateText as any).mockResolvedValue({
|
||||
text: 'done',
|
||||
toolCalls: [],
|
||||
steps: [],
|
||||
response: {
|
||||
headers: {
|
||||
'x-ratelimit-limit-requests': '200',
|
||||
'x-ratelimit-remaining-requests': '30',
|
||||
'x-ratelimit-limit-tokens': '100000',
|
||||
'x-ratelimit-remaining-tokens': '4000',
|
||||
},
|
||||
},
|
||||
});
|
||||
const runtime = new AiSdkKtxLlmRuntime({
|
||||
llmProvider: vertexProvider as any,
|
||||
rateLimitGovernor: { waitForReady, report, maxRetryAttempts: () => 6 } as never,
|
||||
});
|
||||
|
||||
const result = await runtime.runAgentLoop({
|
||||
modelRole: 'candidateExtraction',
|
||||
systemPrompt: '',
|
||||
userPrompt: '',
|
||||
toolSet: {},
|
||||
stepBudget: 10,
|
||||
telemetryTags: {},
|
||||
});
|
||||
|
||||
expect(result.stopReason).toBe('natural');
|
||||
expect(report).toHaveBeenCalledWith({
|
||||
provider: 'vertex',
|
||||
status: 'allowed',
|
||||
rateLimitType: 'tpm',
|
||||
utilization: 0.96,
|
||||
});
|
||||
});
|
||||
|
||||
it('passes abort signals into governor waits and AI SDK generateText calls', async () => {
|
||||
const controller = new AbortController();
|
||||
const waitForReady = vi.fn().mockResolvedValue(undefined);
|
||||
(generateText as any).mockResolvedValue({ text: 'done', toolCalls: [], steps: [] });
|
||||
const runtime = new AiSdkKtxLlmRuntime({
|
||||
llmProvider: llmProvider as any,
|
||||
rateLimitGovernor: { waitForReady, report: vi.fn(), maxRetryAttempts: () => 6 } as never,
|
||||
});
|
||||
|
||||
const result = await runtime.runAgentLoop({
|
||||
modelRole: 'candidateExtraction',
|
||||
systemPrompt: '',
|
||||
userPrompt: '',
|
||||
toolSet: {},
|
||||
stepBudget: 10,
|
||||
telemetryTags: {},
|
||||
abortSignal: controller.signal,
|
||||
});
|
||||
|
||||
expect(result.stopReason).toBe('natural');
|
||||
expect(waitForReady).toHaveBeenCalledWith(controller.signal);
|
||||
expect((generateText as any).mock.calls[0][0].abortSignal).toBe(controller.signal);
|
||||
});
|
||||
|
||||
it('returns metrics with stepCount, per-step boundaries, and aggregate token usage', async () => {
|
||||
(generateText as any).mockImplementation(async (opts: any) => {
|
||||
await opts.onStepFinish({});
|
||||
|
|
|
|||
|
|
@ -9,14 +9,6 @@ async function* stream(messages: SDKMessage[]): AsyncGenerator<SDKMessage, void>
|
|||
}
|
||||
}
|
||||
|
||||
function deferred<T>() {
|
||||
let resolve!: (value: T | PromiseLike<T>) => void;
|
||||
const promise = new Promise<T>((innerResolve) => {
|
||||
resolve = innerResolve;
|
||||
});
|
||||
return { promise, resolve };
|
||||
}
|
||||
|
||||
function initMessage(overrides: Partial<Extract<SDKMessage, { type: 'system'; subtype: 'init' }>> = {}): Extract<
|
||||
SDKMessage,
|
||||
{ type: 'system'; subtype: 'init' }
|
||||
|
|
@ -99,247 +91,6 @@ describe('ClaudeCodeKtxLlmRuntime', () => {
|
|||
});
|
||||
});
|
||||
|
||||
it('waits before Claude Code text generation and reports rate-limit events', async () => {
|
||||
const waitForReady = vi.fn().mockResolvedValue(undefined);
|
||||
const report = vi.fn();
|
||||
const query = vi.fn((_input: any) =>
|
||||
stream([
|
||||
{
|
||||
type: 'rate_limit_event',
|
||||
rate_limit_info: {
|
||||
status: 'allowed_warning',
|
||||
resetsAt: new Date(2_000).toISOString(),
|
||||
rateLimitType: 'five_hour',
|
||||
utilization: 0.91,
|
||||
},
|
||||
} as unknown as SDKMessage,
|
||||
resultMessage({ result: 'ok' }),
|
||||
]),
|
||||
);
|
||||
const runtime = new ClaudeCodeKtxLlmRuntime({
|
||||
projectDir: '/tmp/project',
|
||||
modelSlots: { default: 'sonnet' },
|
||||
query,
|
||||
env: {},
|
||||
rateLimitGovernor: { waitForReady, report, maxRetryAttempts: () => 6 } as never,
|
||||
});
|
||||
|
||||
await expect(runtime.generateText({ role: 'default', prompt: 'hello' })).resolves.toBe('ok');
|
||||
expect(waitForReady).toHaveBeenCalledTimes(1);
|
||||
expect(report).toHaveBeenCalledWith({
|
||||
provider: 'claude-subscription',
|
||||
status: 'warning',
|
||||
resetAtMs: 2_000,
|
||||
rateLimitType: 'five_hour',
|
||||
utilization: 0.91,
|
||||
});
|
||||
});
|
||||
|
||||
it('maps numeric Claude Code reset times from SDK rate-limit events', async () => {
|
||||
const report = vi.fn();
|
||||
const resetAtMs = 1_700_000_000_000;
|
||||
const query = vi.fn((_input: any) =>
|
||||
stream([
|
||||
{
|
||||
type: 'rate_limit_event',
|
||||
rate_limit_info: {
|
||||
status: 'rejected',
|
||||
resetsAt: resetAtMs,
|
||||
rateLimitType: 'five_hour',
|
||||
utilization: 1,
|
||||
},
|
||||
} as unknown as SDKMessage,
|
||||
resultMessage({ result: 'ok' }),
|
||||
]),
|
||||
);
|
||||
const runtime = new ClaudeCodeKtxLlmRuntime({
|
||||
projectDir: '/tmp/project',
|
||||
modelSlots: { default: 'sonnet' },
|
||||
query,
|
||||
env: {},
|
||||
rateLimitGovernor: { waitForReady: vi.fn().mockResolvedValue(undefined), report, maxRetryAttempts: () => 6 } as never,
|
||||
});
|
||||
|
||||
await expect(runtime.generateText({ role: 'default', prompt: 'hello' })).resolves.toBe('ok');
|
||||
|
||||
expect(report).toHaveBeenCalledWith({
|
||||
provider: 'claude-subscription',
|
||||
status: 'rejected',
|
||||
resetAtMs,
|
||||
rateLimitType: 'five_hour',
|
||||
utilization: 1,
|
||||
});
|
||||
});
|
||||
|
||||
it('retries a Claude Code query after an SDK rate-limit result error', async () => {
|
||||
const waitForReady = vi.fn().mockResolvedValue(undefined);
|
||||
const report = vi.fn();
|
||||
const resetAtMs = 1_700_000_000_000;
|
||||
const query = vi
|
||||
.fn()
|
||||
.mockReturnValueOnce(
|
||||
stream([
|
||||
{
|
||||
type: 'rate_limit_event',
|
||||
rate_limit_info: {
|
||||
status: 'rejected',
|
||||
resetsAt: resetAtMs,
|
||||
rateLimitType: 'five_hour',
|
||||
utilization: 1,
|
||||
},
|
||||
} as unknown as SDKMessage,
|
||||
resultMessage({
|
||||
subtype: 'error_during_execution',
|
||||
is_error: true,
|
||||
result: '',
|
||||
errors: ['rate limit retry budget exhausted'],
|
||||
terminal_reason: 'model_error',
|
||||
} as never),
|
||||
]),
|
||||
)
|
||||
.mockReturnValueOnce(stream([resultMessage({ result: 'ok' })]));
|
||||
const runtime = new ClaudeCodeKtxLlmRuntime({
|
||||
projectDir: '/tmp/project',
|
||||
modelSlots: { default: 'sonnet' },
|
||||
query,
|
||||
env: {},
|
||||
rateLimitGovernor: { waitForReady, report, maxRetryAttempts: () => 6 } as never,
|
||||
});
|
||||
|
||||
await expect(runtime.generateText({ role: 'default', prompt: 'hello' })).resolves.toBe('ok');
|
||||
|
||||
expect(query).toHaveBeenCalledTimes(2);
|
||||
expect(waitForReady).toHaveBeenCalledTimes(2);
|
||||
expect(report).toHaveBeenCalledWith({
|
||||
provider: 'claude-subscription',
|
||||
status: 'rejected',
|
||||
resetAtMs,
|
||||
rateLimitType: 'five_hour',
|
||||
utilization: 1,
|
||||
});
|
||||
});
|
||||
|
||||
it('reports Claude Code api retry messages as warning signals', async () => {
|
||||
const report = vi.fn();
|
||||
const query = vi.fn((_input: any) =>
|
||||
stream([
|
||||
{
|
||||
type: 'system',
|
||||
subtype: 'api_retry',
|
||||
retry_delay_ms: 12_000,
|
||||
} as unknown as SDKMessage,
|
||||
resultMessage({ result: 'ok' }),
|
||||
]),
|
||||
);
|
||||
const runtime = new ClaudeCodeKtxLlmRuntime({
|
||||
projectDir: '/tmp/project',
|
||||
modelSlots: { default: 'sonnet' },
|
||||
query,
|
||||
env: {},
|
||||
rateLimitGovernor: { waitForReady: vi.fn().mockResolvedValue(undefined), report, maxRetryAttempts: () => 6 } as never,
|
||||
});
|
||||
|
||||
await runtime.generateText({ role: 'default', prompt: 'hello' });
|
||||
expect(report).toHaveBeenCalledWith({
|
||||
provider: 'claude-subscription',
|
||||
status: 'warning',
|
||||
retryAfterMs: 12_000,
|
||||
rateLimitType: 'api_retry',
|
||||
});
|
||||
});
|
||||
|
||||
it('passes abort signals into Claude Code governor waits', async () => {
|
||||
const controller = new AbortController();
|
||||
const waitForReady = vi.fn().mockResolvedValue(undefined);
|
||||
const query = vi.fn((_input: any) => stream([resultMessage({ result: 'ok' })]));
|
||||
const runtime = new ClaudeCodeKtxLlmRuntime({
|
||||
projectDir: '/tmp/project',
|
||||
modelSlots: { default: 'sonnet' },
|
||||
query,
|
||||
env: {},
|
||||
rateLimitGovernor: { waitForReady, report: vi.fn(), maxRetryAttempts: () => 6 } as never,
|
||||
});
|
||||
|
||||
await expect(runtime.generateText({ role: 'default', prompt: 'hello', abortSignal: controller.signal })).resolves.toBe('ok');
|
||||
|
||||
expect(waitForReady).toHaveBeenCalledWith(controller.signal);
|
||||
});
|
||||
|
||||
it('interrupts an active Claude Code query when the abort signal fires', async () => {
|
||||
const controller = new AbortController();
|
||||
const streamStarted = deferred<void>();
|
||||
const releaseStream = deferred<void>();
|
||||
const interrupt = vi.fn(() => releaseStream.resolve());
|
||||
const queryResult = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
streamStarted.resolve();
|
||||
await releaseStream.promise;
|
||||
yield resultMessage({ result: 'ok' });
|
||||
},
|
||||
interrupt,
|
||||
};
|
||||
const query = vi.fn(() => queryResult as never);
|
||||
const runtime = new ClaudeCodeKtxLlmRuntime({
|
||||
projectDir: '/tmp/project',
|
||||
modelSlots: { default: 'sonnet' },
|
||||
query,
|
||||
env: {},
|
||||
rateLimitGovernor: { waitForReady: vi.fn().mockResolvedValue(undefined), report: vi.fn(), maxRetryAttempts: () => 6 } as never,
|
||||
});
|
||||
|
||||
const pending = runtime.generateText({ role: 'default', prompt: 'hello', abortSignal: controller.signal });
|
||||
await streamStarted.promise;
|
||||
controller.abort();
|
||||
|
||||
await expect(pending).rejects.toThrow(/Aborted/);
|
||||
expect(interrupt).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('throws abort before starting Claude Code query when the signal is already aborted', async () => {
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
const query = vi.fn((_input: any) => stream([resultMessage({ result: 'ok' })]));
|
||||
const runtime = new ClaudeCodeKtxLlmRuntime({
|
||||
projectDir: '/tmp/project',
|
||||
modelSlots: { default: 'sonnet' },
|
||||
query,
|
||||
env: {},
|
||||
rateLimitGovernor: { waitForReady: vi.fn().mockResolvedValue(undefined), report: vi.fn(), maxRetryAttempts: () => 6 } as never,
|
||||
});
|
||||
|
||||
await expect(runtime.generateText({ role: 'default', prompt: 'hello', abortSignal: controller.signal })).rejects.toThrow(/Aborted/);
|
||||
expect(query).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('treats an interrupted Claude Code stream with no result as abort', async () => {
|
||||
const controller = new AbortController();
|
||||
const streamStarted = deferred<void>();
|
||||
const releaseStream = deferred<void>();
|
||||
const interrupt = vi.fn(() => releaseStream.resolve());
|
||||
const queryResult = {
|
||||
async *[Symbol.asyncIterator]() {
|
||||
streamStarted.resolve();
|
||||
await releaseStream.promise;
|
||||
},
|
||||
interrupt,
|
||||
};
|
||||
const query = vi.fn(() => queryResult as never);
|
||||
const runtime = new ClaudeCodeKtxLlmRuntime({
|
||||
projectDir: '/tmp/project',
|
||||
modelSlots: { default: 'sonnet' },
|
||||
query,
|
||||
env: {},
|
||||
rateLimitGovernor: { waitForReady: vi.fn().mockResolvedValue(undefined), report: vi.fn(), maxRetryAttempts: () => 6 } as never,
|
||||
});
|
||||
|
||||
const pending = runtime.generateText({ role: 'default', prompt: 'hello', abortSignal: controller.signal });
|
||||
await streamStarted.promise;
|
||||
controller.abort();
|
||||
|
||||
await expect(pending).rejects.toThrow(/Aborted/);
|
||||
expect(interrupt).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('validates structured output with the caller schema and whitelists the SDK StructuredOutput tool', async () => {
|
||||
const schema = z.object({ answer: z.string() });
|
||||
const query = vi.fn((_input: any) =>
|
||||
|
|
|
|||
|
|
@ -130,150 +130,6 @@ describe('CodexKtxLlmRuntime', () => {
|
|||
).rejects.toThrow('Codex structured output failed validation');
|
||||
});
|
||||
|
||||
it('reports Codex rate-limit failures and retries with opaque backoff', async () => {
|
||||
const waitForReady = vi.fn().mockResolvedValue(undefined);
|
||||
const report = vi.fn();
|
||||
const fakeRunner = {
|
||||
runStreamed: vi
|
||||
.fn()
|
||||
.mockResolvedValueOnce(events([{ type: 'turn.failed', error: { message: '429 rate limit exceeded' } }]))
|
||||
.mockResolvedValueOnce(
|
||||
events([
|
||||
{ type: 'turn.started' },
|
||||
{ type: 'item.completed', item: { type: 'agent_message', text: 'ok' } },
|
||||
{ type: 'turn.completed' },
|
||||
]),
|
||||
),
|
||||
};
|
||||
const runtime = new CodexKtxLlmRuntime({
|
||||
projectDir: '/tmp/project',
|
||||
modelSlots: { default: 'codex' },
|
||||
runner: fakeRunner,
|
||||
rateLimitGovernor: { waitForReady, report, maxRetryAttempts: () => 6 } as never,
|
||||
});
|
||||
|
||||
await expect(runtime.generateText({ role: 'default', prompt: 'hello' })).resolves.toBe('ok');
|
||||
expect(report).toHaveBeenCalledWith({ provider: 'codex', status: 'rejected', rateLimitType: 'opaque' });
|
||||
expect(waitForReady).toHaveBeenCalledTimes(2);
|
||||
expect(fakeRunner.runStreamed).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('reports thrown Codex rate-limit failures and retries with opaque backoff', async () => {
|
||||
const waitForReady = vi.fn().mockResolvedValue(undefined);
|
||||
const report = vi.fn();
|
||||
const fakeRunner = {
|
||||
runStreamed: vi
|
||||
.fn()
|
||||
.mockRejectedValueOnce(new Error('ThreadError: 429 rate limit exceeded'))
|
||||
.mockResolvedValueOnce(
|
||||
events([
|
||||
{ type: 'turn.started' },
|
||||
{ type: 'item.completed', item: { type: 'agent_message', text: 'ok' } },
|
||||
{ type: 'turn.completed' },
|
||||
]),
|
||||
),
|
||||
};
|
||||
const runtime = new CodexKtxLlmRuntime({
|
||||
projectDir: '/tmp/project',
|
||||
modelSlots: { default: 'codex' },
|
||||
runner: fakeRunner,
|
||||
rateLimitGovernor: { waitForReady, report, maxRetryAttempts: () => 6 } as never,
|
||||
});
|
||||
|
||||
await expect(runtime.generateText({ role: 'default', prompt: 'hello' })).resolves.toBe('ok');
|
||||
|
||||
expect(report).toHaveBeenCalledWith({ provider: 'codex', status: 'rejected', rateLimitType: 'opaque' });
|
||||
expect(waitForReady).toHaveBeenCalledTimes(2);
|
||||
expect(fakeRunner.runStreamed).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it('surfaces Codex rate-limit failures without retrying when no governor is present', async () => {
|
||||
const fakeRunner = runner([{ type: 'turn.failed', error: { message: '429 rate limit exceeded' } }]);
|
||||
const runtime = new CodexKtxLlmRuntime({
|
||||
projectDir: '/tmp/project',
|
||||
modelSlots: { default: 'codex' },
|
||||
runner: fakeRunner,
|
||||
});
|
||||
|
||||
await expect(runtime.generateText({ role: 'default', prompt: 'hello' })).rejects.toThrow(/rate limit/i);
|
||||
expect(fakeRunner.runStreamed).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('passes abort signals into Codex text generation and governor waits', async () => {
|
||||
const controller = new AbortController();
|
||||
const waitForReady = vi.fn().mockResolvedValue(undefined);
|
||||
let observedSignal: AbortSignal | undefined;
|
||||
const fakeRunner = {
|
||||
runStreamed: vi.fn(async (input: { signal?: AbortSignal }) => {
|
||||
observedSignal = input.signal;
|
||||
return events([
|
||||
{ type: 'turn.started' },
|
||||
{ type: 'item.completed', item: { type: 'agent_message', text: 'ok' } },
|
||||
{ type: 'turn.completed' },
|
||||
]);
|
||||
}),
|
||||
};
|
||||
const runtime = new CodexKtxLlmRuntime({
|
||||
projectDir: '/tmp/project',
|
||||
modelSlots: { default: 'codex' },
|
||||
runner: fakeRunner,
|
||||
rateLimitGovernor: { waitForReady, report: vi.fn(), maxRetryAttempts: () => 6 } as never,
|
||||
});
|
||||
|
||||
await expect(runtime.generateText({ role: 'default', prompt: 'hello', abortSignal: controller.signal })).resolves.toBe('ok');
|
||||
|
||||
expect(waitForReady).toHaveBeenCalledWith(controller.signal);
|
||||
expect(observedSignal).toBe(controller.signal);
|
||||
});
|
||||
|
||||
it('links the parent abort signal into Codex agent-loop streamed runs', async () => {
|
||||
const controller = new AbortController();
|
||||
let releaseStream!: () => void;
|
||||
const streamRelease = new Promise<void>((resolve) => {
|
||||
releaseStream = resolve;
|
||||
});
|
||||
let markRunnerCalled!: () => void;
|
||||
const runnerCalled = new Promise<void>((resolve) => {
|
||||
markRunnerCalled = resolve;
|
||||
});
|
||||
let observedSignal: AbortSignal | undefined;
|
||||
const fakeRunner = {
|
||||
runStreamed: vi.fn(async (input: { signal?: AbortSignal }) => {
|
||||
observedSignal = input.signal;
|
||||
markRunnerCalled();
|
||||
return (async function* () {
|
||||
await streamRelease;
|
||||
yield { type: 'turn.started' };
|
||||
yield { type: 'item.completed', item: { type: 'agent_message', text: 'ok' } };
|
||||
yield { type: 'turn.completed' };
|
||||
})();
|
||||
}),
|
||||
};
|
||||
const runtime = new CodexKtxLlmRuntime({
|
||||
projectDir: '/tmp/project',
|
||||
modelSlots: { default: 'codex' },
|
||||
runner: fakeRunner,
|
||||
});
|
||||
|
||||
const pending = runtime.runAgentLoop({
|
||||
modelRole: 'default',
|
||||
systemPrompt: '',
|
||||
userPrompt: '',
|
||||
toolSet: {},
|
||||
stepBudget: 10,
|
||||
telemetryTags: {},
|
||||
abortSignal: controller.signal,
|
||||
});
|
||||
|
||||
await runnerCalled;
|
||||
expect(observedSignal).toBeDefined();
|
||||
expect(observedSignal).not.toBe(controller.signal);
|
||||
controller.abort();
|
||||
expect(observedSignal?.aborted).toBe(true);
|
||||
releaseStream();
|
||||
await expect(pending).resolves.toMatchObject({ stopReason: 'natural' });
|
||||
});
|
||||
|
||||
it('starts and closes a temporary MCP server for tool-backed agent loops', async () => {
|
||||
const close = vi.fn(async () => undefined);
|
||||
const startMcpServer = vi.fn(async () => ({
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ import {
|
|||
import {
|
||||
createLocalKtxEmbeddingProviderFromConfig,
|
||||
createLocalKtxLlmProviderFromConfig,
|
||||
createLocalKtxLlmRuntimeFromConfig,
|
||||
resolveLocalKtxEmbeddingConfig,
|
||||
resolveLocalKtxLlmConfig,
|
||||
} from '../../../src/context/llm/local-config.js';
|
||||
|
|
@ -130,64 +129,6 @@ describe('local KTX LLM config', () => {
|
|||
vertexFallbackTo5m: false,
|
||||
});
|
||||
});
|
||||
|
||||
it('passes the rate-limit governor into created runtimes', () => {
|
||||
const rateLimitGovernor = {} as never;
|
||||
const createClaudeCodeRuntime = vi.fn(() => ({
|
||||
generateText: vi.fn(),
|
||||
generateObject: vi.fn(),
|
||||
runAgentLoop: vi.fn(),
|
||||
}));
|
||||
const createCodexRuntime = vi.fn(() => ({
|
||||
generateText: vi.fn(),
|
||||
generateObject: vi.fn(),
|
||||
runAgentLoop: vi.fn(),
|
||||
}));
|
||||
const createAiSdkRuntime = vi.fn(() => ({
|
||||
generateText: vi.fn(),
|
||||
generateObject: vi.fn(),
|
||||
runAgentLoop: vi.fn(),
|
||||
}));
|
||||
const createKtxLlmProvider = vi.fn(() => ({
|
||||
getModel: vi.fn(),
|
||||
getModelByName: vi.fn(),
|
||||
cacheMarker: vi.fn(),
|
||||
repairToolCallHandler: vi.fn(),
|
||||
thinkingProviderOptions: vi.fn(),
|
||||
telemetryConfig: vi.fn(),
|
||||
promptCachingConfig: vi.fn(),
|
||||
activeBackend: vi.fn(() => 'anthropic'),
|
||||
}));
|
||||
|
||||
createLocalKtxLlmRuntimeFromConfig(
|
||||
{
|
||||
provider: { backend: 'claude-code' },
|
||||
models: { default: 'sonnet' },
|
||||
promptCaching: undefined,
|
||||
},
|
||||
{ projectDir: '/tmp/project', env: {}, rateLimitGovernor, createClaudeCodeRuntime },
|
||||
);
|
||||
createLocalKtxLlmRuntimeFromConfig(
|
||||
{
|
||||
provider: { backend: 'codex' },
|
||||
models: { default: 'codex' },
|
||||
promptCaching: undefined,
|
||||
},
|
||||
{ projectDir: '/tmp/project', env: {}, rateLimitGovernor, createCodexRuntime },
|
||||
);
|
||||
createLocalKtxLlmRuntimeFromConfig(
|
||||
{
|
||||
provider: { backend: 'anthropic' },
|
||||
models: { default: 'claude-sonnet-4-6' },
|
||||
promptCaching: undefined,
|
||||
},
|
||||
{ env: {}, rateLimitGovernor, createAiSdkRuntime, createKtxLlmProvider: createKtxLlmProvider as never },
|
||||
);
|
||||
|
||||
expect(createClaudeCodeRuntime).toHaveBeenCalledWith(expect.objectContaining({ rateLimitGovernor }));
|
||||
expect(createCodexRuntime).toHaveBeenCalledWith(expect.objectContaining({ rateLimitGovernor }));
|
||||
expect(createAiSdkRuntime).toHaveBeenCalledWith(expect.objectContaining({ rateLimitGovernor }));
|
||||
});
|
||||
});
|
||||
|
||||
describe('local KTX embedding config', () => {
|
||||
|
|
|
|||
|
|
@ -1,278 +0,0 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
createRateLimitGovernorConfig,
|
||||
RateLimitGovernor,
|
||||
type RateLimitWaitState,
|
||||
} from '../../../src/context/llm/rate-limit-governor.js';
|
||||
|
||||
function testClock(startMs = 1_000) {
|
||||
let nowMs = startMs;
|
||||
return {
|
||||
now: () => nowMs,
|
||||
advance: (ms: number) => {
|
||||
nowMs += ms;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function flushMicrotasks(turns = 10): Promise<void> {
|
||||
for (let i = 0; i < turns; i += 1) {
|
||||
await Promise.resolve();
|
||||
}
|
||||
}
|
||||
|
||||
describe('RateLimitGovernor', () => {
|
||||
it('drops and restores the effective work-unit limit from warning signals', () => {
|
||||
const clock = testClock();
|
||||
const states: RateLimitWaitState[] = [];
|
||||
const governor = new RateLimitGovernor(
|
||||
createRateLimitGovernorConfig({ maxConcurrency: 6, minConcurrencyUnderPressure: 1 }),
|
||||
{ now: clock.now, sleep: async () => undefined, random: () => 0 },
|
||||
);
|
||||
governor.subscribe((state) => states.push(state));
|
||||
|
||||
expect(governor.currentLimit()).toBe(6);
|
||||
governor.report({
|
||||
provider: 'claude-subscription',
|
||||
status: 'warning',
|
||||
utilization: 0.91,
|
||||
rateLimitType: 'five_hour',
|
||||
});
|
||||
expect(governor.currentLimit()).toBe(1);
|
||||
governor.report({
|
||||
provider: 'claude-subscription',
|
||||
status: 'allowed',
|
||||
utilization: 0.2,
|
||||
rateLimitType: 'five_hour',
|
||||
});
|
||||
expect(governor.currentLimit()).toBe(6);
|
||||
expect(states.map((state) => state.kind)).toContain('concurrency_adjusted');
|
||||
});
|
||||
|
||||
it('blocks work slots during a rejected reset window and emits wait states', async () => {
|
||||
const clock = testClock();
|
||||
const states: RateLimitWaitState[] = [];
|
||||
const sleeps: number[] = [];
|
||||
const governor = new RateLimitGovernor(
|
||||
createRateLimitGovernorConfig({ maxConcurrency: 2, waitStateTickMs: 100 }),
|
||||
{
|
||||
now: clock.now,
|
||||
random: () => 0,
|
||||
sleep: async (ms) => {
|
||||
sleeps.push(ms);
|
||||
clock.advance(ms);
|
||||
},
|
||||
},
|
||||
);
|
||||
governor.subscribe((state) => states.push(state));
|
||||
|
||||
governor.report({ provider: 'anthropic-api', status: 'rejected', retryAfterMs: 250, rateLimitType: 'rpm' });
|
||||
const release = await governor.acquireWorkSlot();
|
||||
release();
|
||||
|
||||
expect(sleeps).toEqual([100, 100, 50]);
|
||||
expect(states.some((state) => state.kind === 'wait_started' && state.provider === 'anthropic-api')).toBe(true);
|
||||
expect(states.some((state) => state.kind === 'wait_finished' && state.provider === 'anthropic-api')).toBe(true);
|
||||
});
|
||||
|
||||
it('rejects an interrupted wait without consuming a work slot', async () => {
|
||||
const clock = testClock();
|
||||
let abortListener: (() => void) | undefined;
|
||||
const governor = new RateLimitGovernor(
|
||||
createRateLimitGovernorConfig({ maxConcurrency: 1, waitStateTickMs: 100 }),
|
||||
{
|
||||
now: clock.now,
|
||||
random: () => 0,
|
||||
sleep: async (_ms, signal) =>
|
||||
new Promise<void>((_resolve, reject) => {
|
||||
abortListener = () => reject(new DOMException('Aborted', 'AbortError'));
|
||||
signal?.addEventListener('abort', abortListener, { once: true });
|
||||
}),
|
||||
},
|
||||
);
|
||||
const controller = new AbortController();
|
||||
|
||||
governor.report({
|
||||
provider: 'claude-subscription',
|
||||
status: 'rejected',
|
||||
resetAtMs: 2_000,
|
||||
rateLimitType: 'five_hour',
|
||||
});
|
||||
const pending = governor.acquireWorkSlot(controller.signal);
|
||||
controller.abort();
|
||||
abortListener?.();
|
||||
|
||||
await expect(pending).rejects.toThrow(/Aborted/);
|
||||
expect(governor.activeSlots()).toBe(0);
|
||||
});
|
||||
|
||||
it('rejects an already-aborted ready wait', async () => {
|
||||
const governor = new RateLimitGovernor(
|
||||
createRateLimitGovernorConfig({ maxConcurrency: 1 }),
|
||||
{ sleep: async () => undefined, random: () => 0 },
|
||||
);
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
|
||||
await expect(governor.waitForReady(controller.signal)).rejects.toThrow(/Aborted/);
|
||||
});
|
||||
|
||||
it('rejects an already-aborted work slot without consuming capacity', async () => {
|
||||
const governor = new RateLimitGovernor(
|
||||
createRateLimitGovernorConfig({ maxConcurrency: 1 }),
|
||||
{ sleep: async () => undefined, random: () => 0 },
|
||||
);
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
|
||||
await expect(governor.acquireWorkSlot(controller.signal)).rejects.toThrow(/Aborted/);
|
||||
expect(governor.activeSlots()).toBe(0);
|
||||
});
|
||||
|
||||
it('uses bounded opaque backoff for rejected signals without reset hints', async () => {
|
||||
const clock = testClock();
|
||||
const sleeps: number[] = [];
|
||||
const governor = new RateLimitGovernor(
|
||||
createRateLimitGovernorConfig({
|
||||
maxConcurrency: 1,
|
||||
retry: { maxAttempts: 3, baseDelayMs: 1_000, maxDelayMs: 60_000, jitter: false },
|
||||
}),
|
||||
{
|
||||
now: clock.now,
|
||||
random: () => 0,
|
||||
sleep: async (ms) => {
|
||||
sleeps.push(ms);
|
||||
clock.advance(ms);
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
governor.report({ provider: 'codex', status: 'rejected', rateLimitType: 'opaque' });
|
||||
const release1 = await governor.acquireWorkSlot();
|
||||
release1();
|
||||
governor.report({ provider: 'codex', status: 'rejected', rateLimitType: 'opaque' });
|
||||
const release2 = await governor.acquireWorkSlot();
|
||||
release2();
|
||||
|
||||
expect(sleeps).toEqual([1_000, 2_000]);
|
||||
});
|
||||
|
||||
it('exposes the configured retry budget and disables outer retries when pacing is off', () => {
|
||||
const retry = { maxAttempts: 3, baseDelayMs: 1_000, maxDelayMs: 60_000, jitter: false };
|
||||
const enabled = new RateLimitGovernor(createRateLimitGovernorConfig({ retry }));
|
||||
expect(enabled.maxRetryAttempts()).toBe(3);
|
||||
|
||||
const disabled = new RateLimitGovernor(createRateLimitGovernorConfig({ enabled: false, retry }));
|
||||
expect(disabled.maxRetryAttempts()).toBe(1);
|
||||
});
|
||||
|
||||
it('emits visible wait ticks after a rejected report without a waiting caller', async () => {
|
||||
const clock = testClock();
|
||||
const states: RateLimitWaitState[] = [];
|
||||
const sleeps: number[] = [];
|
||||
const governor = new RateLimitGovernor(
|
||||
createRateLimitGovernorConfig({ maxConcurrency: 4, minConcurrencyUnderPressure: 1, waitStateTickMs: 100 }),
|
||||
{
|
||||
now: clock.now,
|
||||
random: () => 0,
|
||||
sleep: async (ms, signal) => {
|
||||
if (signal?.aborted) {
|
||||
throw new DOMException('Aborted', 'AbortError');
|
||||
}
|
||||
sleeps.push(ms);
|
||||
clock.advance(ms);
|
||||
},
|
||||
},
|
||||
);
|
||||
governor.subscribe((state) => states.push(state));
|
||||
|
||||
governor.report({
|
||||
provider: 'claude-subscription',
|
||||
status: 'rejected',
|
||||
resetAtMs: 1_250,
|
||||
rateLimitType: 'five_hour',
|
||||
});
|
||||
await flushMicrotasks();
|
||||
|
||||
expect(sleeps).toEqual([100, 100, 50]);
|
||||
expect(states).toContainEqual(
|
||||
expect.objectContaining({
|
||||
kind: 'wait_started',
|
||||
provider: 'claude-subscription',
|
||||
rateLimitType: 'five_hour',
|
||||
remainingMs: 250,
|
||||
}),
|
||||
);
|
||||
expect(states.filter((state) => state.kind === 'wait_tick')).toHaveLength(3);
|
||||
expect(states).toContainEqual(
|
||||
expect.objectContaining({
|
||||
kind: 'wait_finished',
|
||||
provider: 'claude-subscription',
|
||||
rateLimitType: 'five_hour',
|
||||
remainingMs: 0,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('does not duplicate countdown sleeps when a work slot waits during the same pause', async () => {
|
||||
const clock = testClock();
|
||||
const states: RateLimitWaitState[] = [];
|
||||
const sleeps: number[] = [];
|
||||
const governor = new RateLimitGovernor(
|
||||
createRateLimitGovernorConfig({ maxConcurrency: 2, waitStateTickMs: 100 }),
|
||||
{
|
||||
now: clock.now,
|
||||
random: () => 0,
|
||||
sleep: async (ms, signal) => {
|
||||
if (signal?.aborted) {
|
||||
throw new DOMException('Aborted', 'AbortError');
|
||||
}
|
||||
sleeps.push(ms);
|
||||
clock.advance(ms);
|
||||
},
|
||||
},
|
||||
);
|
||||
governor.subscribe((state) => states.push(state));
|
||||
|
||||
governor.report({ provider: 'anthropic-api', status: 'rejected', retryAfterMs: 250, rateLimitType: 'rpm' });
|
||||
const pendingRelease = governor.acquireWorkSlot();
|
||||
await flushMicrotasks();
|
||||
const release = await pendingRelease;
|
||||
release();
|
||||
|
||||
expect(sleeps).toEqual([100, 100, 50]);
|
||||
expect(states.filter((state) => state.kind === 'wait_tick')).toHaveLength(3);
|
||||
expect(governor.activeSlots()).toBe(0);
|
||||
});
|
||||
|
||||
it('stops the visible wait ticker when the last subscriber unsubscribes', async () => {
|
||||
const clock = testClock();
|
||||
let abortCount = 0;
|
||||
const governor = new RateLimitGovernor(
|
||||
createRateLimitGovernorConfig({ maxConcurrency: 1, waitStateTickMs: 100 }),
|
||||
{
|
||||
now: clock.now,
|
||||
random: () => 0,
|
||||
sleep: async (_ms, signal) =>
|
||||
new Promise<void>((_resolve, reject) => {
|
||||
signal?.addEventListener(
|
||||
'abort',
|
||||
() => {
|
||||
abortCount += 1;
|
||||
reject(new DOMException('Aborted', 'AbortError'));
|
||||
},
|
||||
{ once: true },
|
||||
);
|
||||
}),
|
||||
},
|
||||
);
|
||||
const unsubscribe = governor.subscribe(() => undefined);
|
||||
|
||||
governor.report({ provider: 'claude-subscription', status: 'rejected', retryAfterMs: 1_000 });
|
||||
await flushMicrotasks(1);
|
||||
unsubscribe();
|
||||
await flushMicrotasks(1);
|
||||
|
||||
expect(abortCount).toBe(1);
|
||||
});
|
||||
});
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
import { access, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
||||
import { access, mkdtemp, readFile, rm } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { Client } from '@modelcontextprotocol/sdk/client/index.js';
|
||||
|
|
@ -7,7 +7,6 @@ import { afterEach, describe, expect, it, vi } from 'vitest';
|
|||
import { createLocalProjectMemoryIngest } from '../../../src/context/memory/local-memory.js';
|
||||
import { detectCaptureSignals } from '../../../src/context/memory/capture-signals.js';
|
||||
import type { MemoryAgentInput } from '../../../src/context/memory/types.js';
|
||||
import { parseKtxProjectConfig, serializeKtxProjectConfig } from '../../../src/context/project/config.js';
|
||||
import { initKtxProject } from '../../../src/context/project/project.js';
|
||||
import { jsonToolResult } from '../../../src/context/mcp/context-tools.js';
|
||||
import { createDefaultKtxMcpServer, createKtxMcpServer } from '../../../src/context/mcp/server.js';
|
||||
|
|
@ -24,12 +23,6 @@ import type {
|
|||
MemoryIngestPort,
|
||||
} from '../../../src/context/mcp/types.js';
|
||||
|
||||
const reportExceptionMock = vi.hoisted(() => vi.fn(async () => {}));
|
||||
|
||||
vi.mock('../../../src/telemetry/exception.js', () => ({
|
||||
reportException: reportExceptionMock,
|
||||
}));
|
||||
|
||||
type RegisteredTool = {
|
||||
name: string;
|
||||
config: {
|
||||
|
|
@ -287,60 +280,6 @@ describe('createKtxMcpServer', () => {
|
|||
expect(io.stderrText()).not.toContain('mcpClientVersion');
|
||||
});
|
||||
|
||||
it('reports MCP tool exceptions with a tool-derived source', async () => {
|
||||
reportExceptionMock.mockClear();
|
||||
vi.stubEnv('ANTHROPIC_API_KEY', 'mcp-anthropic-secret'); // pragma: allowlist secret
|
||||
const fake = makeFakeServer();
|
||||
const io = makeIo();
|
||||
const projectDir = await mkdtemp(join(tmpdir(), 'ktx-mcp-exception-'));
|
||||
try {
|
||||
await initKtxProject({ projectDir });
|
||||
const config = parseKtxProjectConfig(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'));
|
||||
await writeFile(
|
||||
join(projectDir, 'ktx.yaml'),
|
||||
serializeKtxProjectConfig({
|
||||
...config,
|
||||
llm: {
|
||||
...config.llm,
|
||||
provider: {
|
||||
backend: 'anthropic',
|
||||
anthropic: { api_key: 'env:ANTHROPIC_API_KEY' }, // pragma: allowlist secret
|
||||
},
|
||||
models: { default: 'claude-sonnet-4-6' },
|
||||
},
|
||||
}),
|
||||
'utf-8',
|
||||
);
|
||||
|
||||
createKtxMcpServer({
|
||||
server: fake.server,
|
||||
userContext: { userId: 'local-user' },
|
||||
projectDir,
|
||||
io,
|
||||
contextTools: {
|
||||
knowledge: {
|
||||
search: vi.fn<KtxKnowledgeMcpPort['search']>().mockRejectedValue(new Error('wiki failed')),
|
||||
read: vi.fn<KtxKnowledgeMcpPort['read']>().mockResolvedValue(null),
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await expect(getTool(fake.tools, 'wiki_search').handler({ query: 'revenue recognition', limit: 5 })).resolves.toMatchObject({
|
||||
isError: true,
|
||||
});
|
||||
|
||||
expect(reportExceptionMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
context: expect.objectContaining({ source: 'mcp:wiki_search', handled: true, fatal: false }),
|
||||
projectDir,
|
||||
redactionSecrets: expect.arrayContaining(['mcp-anthropic-secret']),
|
||||
}),
|
||||
);
|
||||
} finally {
|
||||
await rm(projectDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('captures the connecting MCP client name and version', async () => {
|
||||
vi.stubEnv('KTX_TELEMETRY_DEBUG', '1');
|
||||
vi.stubEnv('CI', '');
|
||||
|
|
|
|||
|
|
@ -50,17 +50,6 @@ connections:
|
|||
maxConcurrency: 1,
|
||||
failureMode: 'continue',
|
||||
},
|
||||
rateLimit: {
|
||||
enabled: true,
|
||||
throttleThreshold: 0.8,
|
||||
minConcurrencyUnderPressure: 1,
|
||||
retry: {
|
||||
maxAttempts: 6,
|
||||
baseDelayMs: 1_000,
|
||||
maxDelayMs: 60_000,
|
||||
jitter: true,
|
||||
},
|
||||
},
|
||||
profile: false,
|
||||
},
|
||||
agent: {
|
||||
|
|
@ -174,52 +163,6 @@ ingest:
|
|||
expect(parseKtxProjectConfig('ingest:\n profile: json\n').ingest.profile).toBe('json');
|
||||
});
|
||||
|
||||
it('defaults ingest rate-limit settings', () => {
|
||||
const config = buildDefaultKtxProjectConfig();
|
||||
expect(config.ingest.rateLimit).toEqual({
|
||||
enabled: true,
|
||||
throttleThreshold: 0.8,
|
||||
minConcurrencyUnderPressure: 1,
|
||||
retry: {
|
||||
maxAttempts: 6,
|
||||
baseDelayMs: 1_000,
|
||||
maxDelayMs: 60_000,
|
||||
jitter: true,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('validates ingest rate-limit retry settings', () => {
|
||||
const config = parseKtxProjectConfig(`
|
||||
llm:
|
||||
provider:
|
||||
backend: none
|
||||
ingest:
|
||||
rateLimit:
|
||||
enabled: true
|
||||
throttleThreshold: 0.7
|
||||
minConcurrencyUnderPressure: 2
|
||||
maxWaitMs: 300000
|
||||
retry:
|
||||
maxAttempts: 4
|
||||
baseDelayMs: 500
|
||||
maxDelayMs: 30000
|
||||
jitter: false
|
||||
`);
|
||||
expect(config.ingest.rateLimit).toEqual({
|
||||
enabled: true,
|
||||
throttleThreshold: 0.7,
|
||||
minConcurrencyUnderPressure: 2,
|
||||
maxWaitMs: 300_000,
|
||||
retry: {
|
||||
maxAttempts: 4,
|
||||
baseDelayMs: 500,
|
||||
maxDelayMs: 30_000,
|
||||
jitter: false,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it('parses global Vertex LLM config', () => {
|
||||
const config = parseKtxProjectConfig(`
|
||||
llm:
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import { tmpdir } from 'node:os';
|
|||
import { join } from 'node:path';
|
||||
import { buildDefaultKtxProjectConfig, type KtxProjectConfig } from '../src/context/project/config.js';
|
||||
import { initKtxProject } from '../src/context/project/project.js';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import {
|
||||
buildPublicIngestPlan,
|
||||
executePublicIngestTarget,
|
||||
|
|
@ -13,12 +13,6 @@ import {
|
|||
runKtxPublicIngest,
|
||||
} from '../src/public-ingest.js';
|
||||
|
||||
const reportExceptionMock = vi.hoisted(() => vi.fn(async () => {}));
|
||||
|
||||
vi.mock('../src/telemetry/exception.js', () => ({
|
||||
reportException: reportExceptionMock,
|
||||
}));
|
||||
|
||||
/** Count non-overlapping occurrences of `needle` in `haystack`. */
|
||||
function occurrences(haystack: string, needle: string): number {
|
||||
return haystack.split(needle).length - 1;
|
||||
|
|
@ -383,10 +377,6 @@ describe('publicProgressMessage', () => {
|
|||
});
|
||||
|
||||
describe('runKtxPublicIngest', () => {
|
||||
beforeEach(() => {
|
||||
reportExceptionMock.mockClear();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllEnvs();
|
||||
});
|
||||
|
|
@ -1218,104 +1208,6 @@ describe('runKtxPublicIngest', () => {
|
|||
);
|
||||
});
|
||||
|
||||
it('reports foreground runtime preflight exceptions', async () => {
|
||||
const io = makeIo({ isTTY: true, interactive: true });
|
||||
const project = projectWithConnections({
|
||||
warehouse: { driver: 'postgres' },
|
||||
});
|
||||
const ensureRuntime = vi.fn(async (): Promise<ManagedPythonCommandRuntime> => {
|
||||
throw new Error('runtime unavailable');
|
||||
});
|
||||
const runContextBuild = vi.fn(async () => ({ exitCode: 0 }));
|
||||
|
||||
await expect(
|
||||
runKtxPublicIngest(
|
||||
{
|
||||
command: 'run',
|
||||
projectDir: '/tmp/project',
|
||||
targetConnectionId: 'warehouse',
|
||||
all: false,
|
||||
json: false,
|
||||
inputMode: 'auto',
|
||||
queryHistory: 'enabled',
|
||||
cliVersion: '0.2.0',
|
||||
runtimeInstallPolicy: 'prompt',
|
||||
},
|
||||
io.io,
|
||||
{
|
||||
loadProject: vi.fn(async () => project),
|
||||
ensureRuntime,
|
||||
runContextBuild,
|
||||
},
|
||||
),
|
||||
).resolves.toBe(1);
|
||||
|
||||
expect(runContextBuild).not.toHaveBeenCalled();
|
||||
expect(io.stderr()).toContain('runtime unavailable');
|
||||
expect(reportExceptionMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
context: expect.objectContaining({ source: 'ingest runtime', handled: true, fatal: false }),
|
||||
projectDir: '/tmp/project',
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('reports foreground context-build exceptions', async () => {
|
||||
const io = makeIo({ isTTY: true, interactive: true });
|
||||
const config = buildDefaultKtxProjectConfig();
|
||||
const project: KtxPublicIngestProject = {
|
||||
projectDir: '/tmp/project',
|
||||
config: {
|
||||
...config,
|
||||
connections: { warehouse: { driver: 'postgres', password: 'env:INGEST_DB_PASSWORD' } }, // pragma: allowlist secret
|
||||
llm: {
|
||||
...config.llm,
|
||||
provider: {
|
||||
backend: 'anthropic',
|
||||
anthropic: { api_key: 'env:ANTHROPIC_API_KEY' }, // pragma: allowlist secret
|
||||
},
|
||||
models: { default: 'claude-sonnet-4-6' },
|
||||
},
|
||||
},
|
||||
};
|
||||
const runContextBuild = vi.fn(async () => {
|
||||
throw new Error('context build failed');
|
||||
});
|
||||
|
||||
await expect(
|
||||
runKtxPublicIngest(
|
||||
{
|
||||
command: 'run',
|
||||
projectDir: '/tmp/project',
|
||||
targetConnectionId: 'warehouse',
|
||||
all: false,
|
||||
json: false,
|
||||
inputMode: 'auto',
|
||||
queryHistory: 'default',
|
||||
},
|
||||
io.io,
|
||||
{
|
||||
loadProject: vi.fn(async () => project),
|
||||
runContextBuild,
|
||||
env: {
|
||||
...process.env,
|
||||
ANTHROPIC_API_KEY: 'ingest-anthropic-secret', // pragma: allowlist secret
|
||||
INGEST_DB_PASSWORD: 'ingest-db-password', // pragma: allowlist secret
|
||||
},
|
||||
},
|
||||
),
|
||||
).resolves.toBe(1);
|
||||
|
||||
expect(io.stderr()).toContain('context build failed');
|
||||
expect(reportExceptionMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
context: expect.objectContaining({ source: 'ingest context-build', handled: true, fatal: false }),
|
||||
projectDir: '/tmp/project',
|
||||
redactionSecrets: expect.arrayContaining(['ingest-anthropic-secret', 'ingest-db-password']),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('preflights foreground managed embeddings runtime before starting the context-build view', async () => {
|
||||
const io = makeIo({ isTTY: true, interactive: true });
|
||||
const config = buildDefaultKtxProjectConfig();
|
||||
|
|
|
|||
|
|
@ -1,40 +0,0 @@
|
|||
import { describe, expect, it } from 'vitest';
|
||||
import { maskRevealingTail } from '../src/reveal-password-prompt.js';
|
||||
|
||||
const MASK = '▪';
|
||||
|
||||
describe('maskRevealingTail', () => {
|
||||
it('reveals the last `tail` characters of a long value', () => {
|
||||
const value = 'example-token-value-abcd';
|
||||
const masked = maskRevealingTail(value, MASK, 4);
|
||||
expect(masked).toBe(`${MASK.repeat(value.length - 4)}abcd`);
|
||||
expect(masked.endsWith('abcd')).toBe(true);
|
||||
});
|
||||
|
||||
it('keeps the same length as the input so cursor slicing stays aligned', () => {
|
||||
for (const secret of ['', 'a', 'abcdefgh', 'abcdefghijklmnop']) {
|
||||
expect(maskRevealingTail(secret, MASK, 4)).toHaveLength(secret.length);
|
||||
}
|
||||
});
|
||||
|
||||
it('fully masks secrets that are not longer than tail * 2', () => {
|
||||
expect(maskRevealingTail('abcdefgh', MASK, 4)).toBe(MASK.repeat(8));
|
||||
expect(maskRevealingTail('abcd', MASK, 4)).toBe(MASK.repeat(4));
|
||||
expect(maskRevealingTail('ab', MASK, 4)).toBe(MASK.repeat(2));
|
||||
});
|
||||
|
||||
it('reveals the tail once the secret crosses the tail * 2 boundary', () => {
|
||||
// length 9 > 8 → reveal last 4, hide the first 5
|
||||
expect(maskRevealingTail('abcdefghi', MASK, 4)).toBe(`${MASK.repeat(5)}fghi`);
|
||||
});
|
||||
|
||||
it('fully masks an empty value', () => {
|
||||
expect(maskRevealingTail('', MASK, 4)).toBe('');
|
||||
});
|
||||
|
||||
it('honors a custom tail count', () => {
|
||||
// tail 2 reveals only when length > 4
|
||||
expect(maskRevealingTail('abcde', MASK, 2)).toBe(`${MASK.repeat(3)}de`);
|
||||
expect(maskRevealingTail('abcd', MASK, 2)).toBe(MASK.repeat(4));
|
||||
});
|
||||
});
|
||||
|
|
@ -2,19 +2,12 @@ import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
|||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import type { SourceAdapter } from '../src/context/ingest/types.js';
|
||||
import { parseKtxProjectConfig, serializeKtxProjectConfig } from '../src/context/project/config.js';
|
||||
import { initKtxProject } from '../src/context/project/project.js';
|
||||
import type { KtxScanReport } from '../src/context/scan/types.js';
|
||||
import type { LocalScanRunResult, RunLocalScanOptions } from '../src/context/scan/local-scan.js';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { createCliScanProgress, runKtxScan, type KtxScanDeps } from '../src/scan.js';
|
||||
|
||||
const reportExceptionMock = vi.hoisted(() => vi.fn(async () => {}));
|
||||
|
||||
vi.mock('../src/telemetry/exception.js', () => ({
|
||||
reportException: reportExceptionMock,
|
||||
}));
|
||||
|
||||
const sqlServerExtractSchema = vi.hoisted(() =>
|
||||
vi.fn(async (connectionId: string) => ({
|
||||
connectionId,
|
||||
|
|
@ -324,7 +317,6 @@ describe('runKtxScan', () => {
|
|||
|
||||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'ktx-cli-scan-'));
|
||||
reportExceptionMock.mockClear();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
|
|
@ -434,28 +426,7 @@ describe('runKtxScan', () => {
|
|||
it('records the raw errorDetail in scan_completed telemetry when the scan throws', async () => {
|
||||
vi.stubEnv('KTX_TELEMETRY_DEBUG', '1');
|
||||
vi.stubEnv('CI', '');
|
||||
vi.stubEnv('ANTHROPIC_API_KEY', 'anthropic-callsite-secret'); // pragma: allowlist secret
|
||||
vi.stubEnv('DATABASE_URL', 'postgres://svc:scan-db-password@db.example.test/analytics'); // pragma: allowlist secret
|
||||
await initKtxProject({ projectDir: tempDir });
|
||||
const config = parseKtxProjectConfig(await readFile(join(tempDir, 'ktx.yaml'), 'utf-8'));
|
||||
await writeFile(
|
||||
join(tempDir, 'ktx.yaml'),
|
||||
serializeKtxProjectConfig({
|
||||
...config,
|
||||
connections: {
|
||||
warehouse: { driver: 'postgres', url: 'env:DATABASE_URL' },
|
||||
},
|
||||
llm: {
|
||||
...config.llm,
|
||||
provider: {
|
||||
backend: 'anthropic',
|
||||
anthropic: { api_key: 'env:ANTHROPIC_API_KEY' }, // pragma: allowlist secret
|
||||
},
|
||||
models: { default: 'claude-sonnet-4-6' },
|
||||
},
|
||||
}),
|
||||
'utf-8',
|
||||
);
|
||||
const runLocalScan = vi.fn(async (): Promise<LocalScanRunResult> => {
|
||||
const error = new Error('introspection timed out');
|
||||
(error as { code?: unknown }).code = 'ETIMEDOUT';
|
||||
|
|
@ -481,17 +452,6 @@ describe('runKtxScan', () => {
|
|||
expect(io.stderr()).toContain('"event":"scan_completed"');
|
||||
expect(io.stderr()).toContain('"outcome":"error"');
|
||||
expect(io.stderr()).toContain('"errorDetail":"ETIMEDOUT: introspection timed out"');
|
||||
expect(reportExceptionMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
context: expect.objectContaining({ source: 'scan run', handled: true, fatal: false }),
|
||||
projectDir: tempDir,
|
||||
redactionSecrets: expect.arrayContaining([
|
||||
'anthropic-callsite-secret',
|
||||
'postgres://svc:scan-db-password@db.example.test/analytics', // pragma: allowlist secret
|
||||
'scan-db-password',
|
||||
]),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('passes KTX daemon options to local ingest adapters when no explicit daemon URL is set', async () => {
|
||||
|
|
|
|||
|
|
@ -2654,7 +2654,6 @@ describe('setup databases step', () => {
|
|||
consideredRoleCount: 2,
|
||||
skipped: null,
|
||||
warnings: [],
|
||||
parseFailedTemplateIds: [],
|
||||
}));
|
||||
|
||||
const result = await runKtxSetupDatabasesStep(
|
||||
|
|
@ -2707,54 +2706,6 @@ describe('setup databases step', () => {
|
|||
expect(io.stdout()).toContain('svc_loader');
|
||||
});
|
||||
|
||||
it('collapses query-history parse failures to a count and lists ids only with --debug', async () => {
|
||||
const io = makeIo();
|
||||
const queryHistoryFilterPicker = vi.fn(async () => ({
|
||||
excludedRoles: [],
|
||||
consideredRoleCount: 1,
|
||||
skipped: { reason: 'no-in-scope-history' as const },
|
||||
warnings: [],
|
||||
parseFailedTemplateIds: ['111', '222'],
|
||||
}));
|
||||
|
||||
const result = await runKtxSetupDatabasesStep(
|
||||
{
|
||||
projectDir: tempDir,
|
||||
inputMode: 'disabled',
|
||||
debug: true,
|
||||
yes: true,
|
||||
databaseDrivers: ['postgres'],
|
||||
databaseConnectionId: 'warehouse',
|
||||
databaseUrl: 'env:DATABASE_URL',
|
||||
databaseSchemas: ['public'],
|
||||
enableQueryHistory: true,
|
||||
skipDatabases: false,
|
||||
},
|
||||
io.io,
|
||||
{
|
||||
testConnection: vi.fn(async () => 0),
|
||||
scanConnection: vi.fn(async () => 0),
|
||||
historicSqlReadinessProbe: vi.fn(async () => {
|
||||
const runner = fakeHistoricSqlRunner('postgres', 'pg_stat_statements');
|
||||
return {
|
||||
ok: true as const,
|
||||
dialect: 'postgres' as const,
|
||||
runner,
|
||||
result: { pgServerVersion: 'PostgreSQL 16.4', warnings: [], info: [] },
|
||||
};
|
||||
}),
|
||||
queryHistoryFilterPicker,
|
||||
createQueryHistoryLlmRuntime: vi.fn(() => null),
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.status).toBe('ready');
|
||||
expect(io.stdout()).toContain('Skipped 2 query templates ktx could not parse');
|
||||
expect(io.stdout()).not.toContain('111');
|
||||
expect(io.stdout()).not.toContain('222');
|
||||
expect(io.stderr()).toContain('could not parse 2 template(s): 111, 222');
|
||||
});
|
||||
|
||||
it('lets interactive setup skip applying derived filters', async () => {
|
||||
const io = makeIo();
|
||||
const prompts = makePromptAdapter({
|
||||
|
|
@ -2792,7 +2743,6 @@ describe('setup databases step', () => {
|
|||
consideredRoleCount: 2,
|
||||
skipped: null,
|
||||
warnings: [],
|
||||
parseFailedTemplateIds: [],
|
||||
})),
|
||||
createQueryHistoryLlmRuntime: vi.fn(() => null),
|
||||
},
|
||||
|
|
@ -2861,7 +2811,6 @@ describe('setup databases step', () => {
|
|||
consideredRoleCount: 2,
|
||||
skipped: { reason: 'user-block-present' as const },
|
||||
warnings: [],
|
||||
parseFailedTemplateIds: [],
|
||||
})),
|
||||
createQueryHistoryLlmRuntime: vi.fn(() => null),
|
||||
},
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ const mocks = vi.hoisted(() => {
|
|||
autocomplete: vi.fn(),
|
||||
autocompleteMultiselect: vi.fn(),
|
||||
note: vi.fn(),
|
||||
revealPassword: vi.fn(),
|
||||
password: vi.fn(),
|
||||
select: vi.fn(),
|
||||
text: vi.fn(),
|
||||
withSetupInterruptConfirmation: vi.fn((prompt: () => Promise<unknown>) => prompt()),
|
||||
|
|
@ -34,14 +34,11 @@ vi.mock('@clack/prompts', () => ({
|
|||
autocomplete: mocks.autocomplete,
|
||||
autocompleteMultiselect: mocks.autocompleteMultiselect,
|
||||
note: mocks.note,
|
||||
password: mocks.password,
|
||||
select: mocks.select,
|
||||
text: mocks.text,
|
||||
}));
|
||||
|
||||
vi.mock('../src/reveal-password-prompt.js', () => ({
|
||||
revealPassword: mocks.revealPassword,
|
||||
}));
|
||||
|
||||
vi.mock('../src/setup-interrupt.js', () => ({
|
||||
withSetupInterruptConfirmation: mocks.withSetupInterruptConfirmation,
|
||||
}));
|
||||
|
|
@ -57,7 +54,7 @@ describe('setup prompt adapter', () => {
|
|||
mocks.autocomplete.mockReset();
|
||||
mocks.autocompleteMultiselect.mockReset();
|
||||
mocks.note.mockReset();
|
||||
mocks.revealPassword.mockReset();
|
||||
mocks.password.mockReset();
|
||||
mocks.select.mockReset();
|
||||
mocks.text.mockReset();
|
||||
mocks.withSetupInterruptConfirmation.mockClear();
|
||||
|
|
@ -99,7 +96,7 @@ describe('setup prompt adapter', () => {
|
|||
|
||||
it('decorates text and password prompts with setup navigation copy', async () => {
|
||||
mocks.text.mockResolvedValueOnce('analytics-ktx');
|
||||
mocks.revealPassword.mockResolvedValueOnce('secret');
|
||||
mocks.password.mockResolvedValueOnce('secret');
|
||||
const adapter = createKtxSetupPromptAdapter({ selectCancelValue: 'back' });
|
||||
|
||||
await expect(adapter.text({ message: 'Project folder path', placeholder: './analytics-ktx' })).resolves.toBe(
|
||||
|
|
@ -111,7 +108,7 @@ describe('setup prompt adapter', () => {
|
|||
message: 'Project folder path\n│ Press Escape to go back.\n│',
|
||||
placeholder: './analytics-ktx',
|
||||
});
|
||||
expect(mocks.revealPassword).toHaveBeenCalledWith({
|
||||
expect(mocks.password).toHaveBeenCalledWith({
|
||||
message: 'Anthropic API key\n│ Press Escape to go back.\n│',
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -447,8 +447,8 @@ describe('setup sources step', () => {
|
|||
expect(testPrompts.select).toHaveBeenCalledWith({
|
||||
message: 'Which Notion pages should KTX ingest?',
|
||||
options: [
|
||||
{ value: 'all_accessible', label: 'All pages the integration can access' },
|
||||
{ value: 'selected_roots', label: 'Specific pages and their subpages (choose them in a picker)' },
|
||||
{ value: 'all_accessible', label: 'All pages the integration can access' },
|
||||
{ value: 'back', label: 'Back' },
|
||||
],
|
||||
});
|
||||
|
|
@ -891,8 +891,8 @@ describe('setup sources step', () => {
|
|||
expect(testPrompts.select).toHaveBeenCalledWith({
|
||||
message: 'This repo requires authentication.',
|
||||
options: [
|
||||
{ value: 'paste', label: 'Paste a token and save it as a local secret file' },
|
||||
{ value: 'env', label: 'Use GITHUB_TOKEN from the environment' },
|
||||
{ value: 'paste', label: 'Paste a token and save it as a local secret file' },
|
||||
{ value: 'skip', label: 'Skip — try without authentication' },
|
||||
{ value: 'back', label: 'Back' },
|
||||
],
|
||||
|
|
@ -1407,8 +1407,8 @@ describe('setup sources step', () => {
|
|||
message: 'How should KTX find your Notion integration token?',
|
||||
options: [
|
||||
{ value: 'keep', label: 'Keep existing credential' },
|
||||
{ value: 'paste', label: 'Paste a key and save it as a local secret file' },
|
||||
{ value: 'env', label: 'Use NOTION_TOKEN from the environment' },
|
||||
{ value: 'paste', label: 'Paste a key and save it as a local secret file' },
|
||||
{ value: 'back', label: 'Back' },
|
||||
],
|
||||
});
|
||||
|
|
@ -1476,8 +1476,8 @@ describe('setup sources step', () => {
|
|||
message: 'How should KTX find your Metabase API key?',
|
||||
options: [
|
||||
{ value: 'keep', label: 'Keep existing credential' },
|
||||
{ value: 'paste', label: 'Paste a key and save it as a local secret file' },
|
||||
{ value: 'env', label: 'Use METABASE_API_KEY from the environment' },
|
||||
{ value: 'paste', label: 'Paste a key and save it as a local secret file' },
|
||||
{ value: 'back', label: 'Back' },
|
||||
],
|
||||
});
|
||||
|
|
@ -1582,8 +1582,8 @@ describe('setup sources step', () => {
|
|||
message: 'This MetricFlow repo requires authentication.',
|
||||
options: [
|
||||
{ value: 'keep', label: 'Keep existing credential' },
|
||||
{ value: 'paste', label: 'Paste a token and save it as a local secret file' },
|
||||
{ value: 'env', label: 'Use GITHUB_TOKEN from the environment' },
|
||||
{ value: 'paste', label: 'Paste a token and save it as a local secret file' },
|
||||
{ value: 'skip', label: 'Skip — try without authentication' },
|
||||
{ value: 'back', label: 'Back' },
|
||||
],
|
||||
|
|
@ -1627,7 +1627,7 @@ describe('setup sources step', () => {
|
|||
expect(testPrompts.select).toHaveBeenCalledWith({
|
||||
message: '1 context source configured (dbt-main). Add another?',
|
||||
options: [
|
||||
{ value: 'done', label: 'Done adding context sources' },
|
||||
{ value: 'done', label: 'Done — continue to context build' },
|
||||
{ value: 'edit', label: 'Edit an existing context source' },
|
||||
{ value: 'add', label: 'Add another context source' },
|
||||
],
|
||||
|
|
|
|||
|
|
@ -1,19 +1,12 @@
|
|||
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
|
||||
import { mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { stripVTControlCharacters } from 'node:util';
|
||||
import Database from 'better-sqlite3';
|
||||
import { parseKtxProjectConfig, serializeKtxProjectConfig } from '../src/context/project/config.js';
|
||||
import { initKtxProject } from '../src/context/project/project.js';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { runKtxSl } from '../src/sl.js';
|
||||
|
||||
const reportExceptionMock = vi.hoisted(() => vi.fn(async () => {}));
|
||||
|
||||
vi.mock('../src/telemetry/exception.js', () => ({
|
||||
reportException: reportExceptionMock,
|
||||
}));
|
||||
|
||||
const ORDERS_YAML = [
|
||||
'name: orders',
|
||||
'table: public.orders',
|
||||
|
|
@ -68,7 +61,6 @@ describe('runKtxSl', () => {
|
|||
|
||||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'ktx-cli-sl-'));
|
||||
reportExceptionMock.mockClear();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
|
|
@ -359,12 +351,6 @@ describe('runKtxSl', () => {
|
|||
|
||||
expect(validateIo.stdout()).toBe('');
|
||||
expect(validateIo.stderr()).toBe('Semantic-layer source "missing_orders" was not found\n');
|
||||
expect(reportExceptionMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
context: expect.objectContaining({ source: 'sl validate', handled: true, fatal: false }),
|
||||
projectDir,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('keeps scoped validation not-found wording', async () => {
|
||||
|
|
@ -566,53 +552,6 @@ joins: []
|
|||
expect(stderr.write).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('reports sl query exceptions at the query catch boundary', async () => {
|
||||
vi.stubEnv('ANTHROPIC_API_KEY', 'sl-anthropic-secret'); // pragma: allowlist secret
|
||||
const projectDir = join(tempDir, 'missing-query-input');
|
||||
await seedSlSource({ projectDir });
|
||||
const config = parseKtxProjectConfig(await readFile(join(projectDir, 'ktx.yaml'), 'utf-8'));
|
||||
await writeFile(
|
||||
join(projectDir, 'ktx.yaml'),
|
||||
serializeKtxProjectConfig({
|
||||
...config,
|
||||
llm: {
|
||||
...config.llm,
|
||||
provider: {
|
||||
backend: 'anthropic',
|
||||
anthropic: { api_key: 'env:ANTHROPIC_API_KEY' }, // pragma: allowlist secret
|
||||
},
|
||||
models: { default: 'claude-sonnet-4-6' },
|
||||
},
|
||||
}),
|
||||
'utf-8',
|
||||
);
|
||||
const io = makeIo();
|
||||
|
||||
await expect(
|
||||
runKtxSl(
|
||||
{
|
||||
command: 'query',
|
||||
projectDir,
|
||||
connectionId: 'warehouse',
|
||||
format: 'json',
|
||||
execute: false,
|
||||
cliVersion: '0.2.0',
|
||||
runtimeInstallPolicy: 'auto',
|
||||
},
|
||||
io.io,
|
||||
),
|
||||
).resolves.toBe(1);
|
||||
|
||||
expect(io.stderr()).toContain('sl query requires query input');
|
||||
expect(reportExceptionMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
context: expect.objectContaining({ source: 'sl query', handled: true, fatal: false }),
|
||||
projectDir,
|
||||
redactionSecrets: expect.arrayContaining(['sl-anthropic-secret']),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('emits debug telemetry for sl query without project paths', async () => {
|
||||
vi.stubEnv('KTX_TELEMETRY_DEBUG', '1');
|
||||
vi.stubEnv('CI', '');
|
||||
|
|
|
|||
|
|
@ -8,12 +8,6 @@ import type { SqlAnalysisPort } from '../src/context/sql-analysis/ports.js';
|
|||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { runKtxSql } from '../src/sql.js';
|
||||
|
||||
const reportExceptionMock = vi.hoisted(() => vi.fn(async () => {}));
|
||||
|
||||
vi.mock('../src/telemetry/exception.js', () => ({
|
||||
reportException: reportExceptionMock,
|
||||
}));
|
||||
|
||||
function makeIo(options: { isTTY?: boolean } = {}) {
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
|
|
@ -82,7 +76,6 @@ describe('runKtxSql', () => {
|
|||
|
||||
beforeEach(async () => {
|
||||
tempDir = await mkdtemp(join(tmpdir(), 'ktx-cli-sql-'));
|
||||
reportExceptionMock.mockClear();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
|
|
@ -243,10 +236,9 @@ describe('runKtxSql', () => {
|
|||
});
|
||||
|
||||
it('rejects non-read-only SQL before executing connector SQL', async () => {
|
||||
vi.stubEnv('SQL_DB_PASSWORD', 'sql-db-password'); // pragma: allowlist secret
|
||||
const projectDir = join(tempDir, 'project');
|
||||
await initKtxProject({ projectDir });
|
||||
await writeConnections(projectDir, { warehouse: { driver: 'postgres', password: 'env:SQL_DB_PASSWORD' } }); // pragma: allowlist secret
|
||||
await writeConnections(projectDir, { warehouse: { driver: 'sqlite', path: 'warehouse.db' } });
|
||||
const connector = makeConnector();
|
||||
const io = makeIo();
|
||||
|
||||
|
|
@ -273,13 +265,6 @@ describe('runKtxSql', () => {
|
|||
expect(connector.executeReadOnly).not.toHaveBeenCalled();
|
||||
expect(connector.cleanup).not.toHaveBeenCalled();
|
||||
expect(io.stderr()).toContain('SQL contains read/write operation: Delete');
|
||||
expect(reportExceptionMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
context: expect.objectContaining({ source: 'sql run', handled: true, fatal: false }),
|
||||
projectDir,
|
||||
redactionSecrets: expect.arrayContaining(['sql-db-password']),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it('rejects missing connections', async () => {
|
||||
|
|
|
|||
|
|
@ -37,7 +37,6 @@ describe('telemetry event schemas', () => {
|
|||
'daemon_stopped',
|
||||
'sl_plan_completed',
|
||||
'sql_gen_completed',
|
||||
'query_history_filter_completed',
|
||||
]);
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -1,150 +0,0 @@
|
|||
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { createServer, type IncomingMessage } from 'node:http';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { gunzipSync } from 'node:zlib';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import type { KtxCliIo } from '../../src/cli-runtime.js';
|
||||
import { __resetTelemetryEmitterForTests } from '../../src/telemetry/emitter.js';
|
||||
import {
|
||||
__resetTelemetryExceptionStateForTests,
|
||||
reportException,
|
||||
} from '../../src/telemetry/exception.js';
|
||||
|
||||
function makeIo(): KtxCliIo {
|
||||
return {
|
||||
stdout: { write: () => {} },
|
||||
stderr: { write: () => {} },
|
||||
};
|
||||
}
|
||||
|
||||
async function body(req: IncomingMessage): Promise<string> {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of req) {
|
||||
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
||||
}
|
||||
const raw = Buffer.concat(chunks);
|
||||
return req.headers['content-encoding'] === 'gzip' ? gunzipSync(raw).toString('utf-8') : raw.toString('utf-8');
|
||||
}
|
||||
|
||||
async function withCaptureServer<T>(run: (url: string, payloads: unknown[]) => Promise<T>): Promise<T> {
|
||||
const payloads: unknown[] = [];
|
||||
const server = createServer(async (req, res) => {
|
||||
if (req.method === 'POST') {
|
||||
payloads.push(JSON.parse(await body(req)));
|
||||
}
|
||||
res.statusCode = 200;
|
||||
res.setHeader('content-type', 'application/json');
|
||||
res.end('{}');
|
||||
});
|
||||
await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve));
|
||||
const address = server.address();
|
||||
if (!address || typeof address === 'string') {
|
||||
throw new Error('test server did not bind to a TCP port');
|
||||
}
|
||||
try {
|
||||
return await run(`http://127.0.0.1:${address.port}`, payloads);
|
||||
} finally {
|
||||
await new Promise<void>((resolve) => server.close(() => resolve()));
|
||||
}
|
||||
}
|
||||
|
||||
function findExceptionEvent(payloads: unknown[]): Record<string, unknown> {
|
||||
for (const payload of payloads) {
|
||||
if (typeof payload !== 'object' || payload === null) {
|
||||
continue;
|
||||
}
|
||||
const record = payload as Record<string, unknown>;
|
||||
const batch = Array.isArray(record.batch) ? record.batch : [record];
|
||||
for (const item of batch) {
|
||||
if (typeof item === 'object' && item !== null && (item as Record<string, unknown>).event === '$exception') {
|
||||
return item as Record<string, unknown>;
|
||||
}
|
||||
}
|
||||
}
|
||||
throw new Error(`No $exception payload found: ${JSON.stringify(payloads)}`);
|
||||
}
|
||||
|
||||
describe('prepared Node exception payload', () => {
|
||||
let homeDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
homeDir = await mkdtemp(join(tmpdir(), 'ktx-node-exception-payload-'));
|
||||
await mkdir(join(homeDir, '.ktx'), { recursive: true });
|
||||
await writeFile(
|
||||
join(homeDir, '.ktx', 'telemetry.json'),
|
||||
`${JSON.stringify({
|
||||
installId: '00000000-0000-4000-8000-000000000000',
|
||||
enabled: true,
|
||||
createdAt: '2026-06-05T00:00:00.000Z',
|
||||
})}\n`,
|
||||
'utf-8',
|
||||
);
|
||||
vi.stubEnv('HOME', homeDir);
|
||||
vi.stubEnv('CI', '');
|
||||
vi.stubEnv('KTX_TELEMETRY_DISABLED', '');
|
||||
vi.stubEnv('DO_NOT_TRACK', '');
|
||||
__resetTelemetryEmitterForTests();
|
||||
__resetTelemetryExceptionStateForTests();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
vi.unstubAllEnvs();
|
||||
await rm(homeDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('sends projectId, omits $groups, and redacts the serialized exception list', async () => {
|
||||
await withCaptureServer(async (endpoint, payloads) => {
|
||||
vi.stubEnv('KTX_TELEMETRY_ENDPOINT', endpoint);
|
||||
const projectDir = join(homeDir, 'project');
|
||||
const snapshotSecret = ['plain', 'secret', 'value'].join('-');
|
||||
const dbPassword = ['db', 'url', 'secret'].join('-');
|
||||
const authToken = ['abc', '123'].join('');
|
||||
const error = new Error(
|
||||
`${snapshotSecret} postgres://svc:${dbPassword}@db.example.test/analytics Authorization: Basic ${authToken}`,
|
||||
);
|
||||
|
||||
await reportException({
|
||||
error,
|
||||
context: { source: 'scan run', handled: true, fatal: false },
|
||||
io: makeIo(),
|
||||
packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' },
|
||||
projectDir,
|
||||
immediate: true,
|
||||
redactionSecrets: [snapshotSecret],
|
||||
});
|
||||
|
||||
const event = findExceptionEvent(payloads);
|
||||
const properties = event.properties as Record<string, unknown>;
|
||||
expect(properties.projectId).toMatch(/^[a-f0-9]{64}$/);
|
||||
expect(properties.$groups).toBeUndefined();
|
||||
expect(JSON.stringify(properties.$exception_list)).toContain('[redacted]');
|
||||
expect(JSON.stringify(properties.$exception_list)).not.toContain(snapshotSecret);
|
||||
expect(JSON.stringify(properties.$exception_list)).not.toContain(dbPassword);
|
||||
expect(JSON.stringify(properties.$exception_list)).not.toContain(authToken);
|
||||
for (const key of [
|
||||
'argv',
|
||||
'args',
|
||||
'env',
|
||||
'environment',
|
||||
'sql',
|
||||
'query',
|
||||
'prompt',
|
||||
'mcpArguments',
|
||||
'tableName',
|
||||
'schemaName',
|
||||
'columnName',
|
||||
'databaseUrl',
|
||||
'connectionString',
|
||||
'url',
|
||||
'password',
|
||||
'token',
|
||||
'apiKey',
|
||||
'authorization',
|
||||
]) {
|
||||
expect(properties).not.toHaveProperty(key);
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -1,456 +0,0 @@
|
|||
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import type { KtxCliIo } from '../../src/cli-runtime.js';
|
||||
import { __resetTelemetryEmitterForTests } from '../../src/telemetry/emitter.js';
|
||||
import {
|
||||
__resetTelemetryExceptionStateForTests,
|
||||
reportException,
|
||||
} from '../../src/telemetry/exception.js';
|
||||
|
||||
const captures: unknown[] = [];
|
||||
const immediateCaptures: unknown[] = [];
|
||||
const shutdown = vi.fn(async () => {});
|
||||
|
||||
vi.mock('posthog-node', () => ({
|
||||
PostHog: vi.fn(function PostHog() {
|
||||
return {
|
||||
captureException: (
|
||||
error: unknown,
|
||||
distinctId?: string,
|
||||
properties?: Record<string, unknown>,
|
||||
) => {
|
||||
captures.push({ error, distinctId, properties });
|
||||
},
|
||||
captureExceptionImmediate: async (
|
||||
error: unknown,
|
||||
distinctId?: string,
|
||||
properties?: Record<string, unknown>,
|
||||
) => {
|
||||
immediateCaptures.push({ error, distinctId, properties });
|
||||
},
|
||||
capture: vi.fn(),
|
||||
shutdown,
|
||||
};
|
||||
}),
|
||||
}));
|
||||
|
||||
function makeIo(): { io: KtxCliIo; stderr: () => string } {
|
||||
let stderr = '';
|
||||
return {
|
||||
io: {
|
||||
stdout: { write: () => {} },
|
||||
stderr: {
|
||||
write: (chunk) => {
|
||||
stderr += chunk;
|
||||
},
|
||||
},
|
||||
},
|
||||
stderr: () => stderr,
|
||||
};
|
||||
}
|
||||
|
||||
async function writeIdentity(homeDir: string, enabled = true): Promise<void> {
|
||||
const path = join(homeDir, '.ktx', 'telemetry.json');
|
||||
await mkdir(join(homeDir, '.ktx'), { recursive: true });
|
||||
await writeFile(
|
||||
path,
|
||||
`${JSON.stringify({
|
||||
installId: '00000000-0000-4000-8000-000000000000',
|
||||
enabled,
|
||||
createdAt: '2026-06-05T00:00:00.000Z',
|
||||
})}\n`,
|
||||
'utf-8',
|
||||
);
|
||||
}
|
||||
|
||||
describe('reportException', () => {
|
||||
let homeDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
homeDir = await mkdtemp(join(tmpdir(), 'ktx-exception-'));
|
||||
await writeIdentity(homeDir);
|
||||
vi.stubEnv('HOME', homeDir);
|
||||
vi.stubEnv('CI', '');
|
||||
vi.stubEnv('KTX_TELEMETRY_DISABLED', '');
|
||||
vi.stubEnv('DO_NOT_TRACK', '');
|
||||
captures.length = 0;
|
||||
immediateCaptures.length = 0;
|
||||
shutdown.mockClear();
|
||||
__resetTelemetryEmitterForTests();
|
||||
__resetTelemetryExceptionStateForTests();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
vi.unstubAllEnvs();
|
||||
await rm(homeDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('honors telemetry kill switches', async () => {
|
||||
vi.stubEnv('KTX_TELEMETRY_DISABLED', '1');
|
||||
const { io } = makeIo();
|
||||
|
||||
await reportException({
|
||||
error: new Error('boom'),
|
||||
context: { source: 'scan run', handled: true, fatal: false },
|
||||
io,
|
||||
packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' },
|
||||
projectDir: join(homeDir, 'project'),
|
||||
});
|
||||
|
||||
expect(captures).toEqual([]);
|
||||
expect(immediateCaptures).toEqual([]);
|
||||
});
|
||||
|
||||
it('prints debug payloads without sending', async () => {
|
||||
vi.stubEnv('KTX_TELEMETRY_DEBUG', '1');
|
||||
vi.stubEnv('KTX_TELEMETRY_DISABLED', '1');
|
||||
const { io, stderr } = makeIo();
|
||||
|
||||
await reportException({
|
||||
error: new Error('debug boom'),
|
||||
context: { source: 'scan run', handled: true, fatal: false },
|
||||
io,
|
||||
packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' },
|
||||
projectDir: join(homeDir, 'project'),
|
||||
});
|
||||
|
||||
expect(stderr()).toContain('[telemetry-exception]');
|
||||
expect(stderr()).toContain('"source":"scan run"');
|
||||
expect(captures).toEqual([]);
|
||||
});
|
||||
|
||||
it('sends projectId as a property and omits $groups for Node exceptions', async () => {
|
||||
const { io } = makeIo();
|
||||
|
||||
await reportException({
|
||||
error: new Error('project boom'),
|
||||
context: { source: 'sql run', handled: true, fatal: false },
|
||||
io,
|
||||
packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' },
|
||||
projectDir: join(homeDir, 'project'),
|
||||
});
|
||||
|
||||
expect(captures).toHaveLength(1);
|
||||
expect(captures[0]).toMatchObject({
|
||||
distinctId: '00000000-0000-4000-8000-000000000000',
|
||||
properties: {
|
||||
source: 'sql run',
|
||||
handled: true,
|
||||
fatal: false,
|
||||
cliVersion: '0.0.0-test',
|
||||
runtime: 'node',
|
||||
},
|
||||
});
|
||||
expect(
|
||||
(captures[0] as { properties: Record<string, unknown> }).properties.projectId,
|
||||
).toMatch(/^[a-f0-9]{64}$/);
|
||||
expect((captures[0] as { properties: Record<string, unknown> }).properties.$groups).toBeUndefined();
|
||||
});
|
||||
|
||||
it('uses captureExceptionImmediate for fatal reports', async () => {
|
||||
const { io } = makeIo();
|
||||
|
||||
await reportException({
|
||||
error: new Error('fatal boom'),
|
||||
context: { source: 'uncaughtException', handled: false, fatal: true },
|
||||
io,
|
||||
packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' },
|
||||
immediate: true,
|
||||
});
|
||||
|
||||
expect(immediateCaptures).toHaveLength(1);
|
||||
expect(captures).toEqual([]);
|
||||
});
|
||||
|
||||
it('redacts snapshot secrets and static credential patterns from message and cause', async () => {
|
||||
const { io } = makeIo();
|
||||
const cause = new Error('cause has sk-live-fixture-value and Authorization: Bearer token-123');
|
||||
const error = new Error('message has sk-live-fixture-value and password=hunter2', { cause });
|
||||
|
||||
await reportException({
|
||||
error,
|
||||
context: { source: 'connection test', handled: true, fatal: false },
|
||||
io,
|
||||
packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' },
|
||||
redactionSecrets: ['sk-live-fixture-value'],
|
||||
});
|
||||
|
||||
const sent = captures[0] as { error: Error & { cause?: Error } };
|
||||
expect(sent.error.message).toContain('[redacted]');
|
||||
expect(sent.error.message).not.toContain('sk-live-fixture-value');
|
||||
expect(sent.error.message).not.toContain('hunter2');
|
||||
expect(sent.error.cause?.message).not.toContain('token-123');
|
||||
});
|
||||
|
||||
it('redacts URL userinfo credentials and non-bearer authorization values', async () => {
|
||||
const { io } = makeIo();
|
||||
const error = new Error(
|
||||
'connect postgres://svc:db-url-secret@db.example.test/analytics Authorization: Basic abc123', // pragma: allowlist secret
|
||||
);
|
||||
|
||||
await reportException({
|
||||
error,
|
||||
context: { source: 'connection test', handled: true, fatal: false },
|
||||
io,
|
||||
packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' },
|
||||
});
|
||||
|
||||
const sent = captures[0] as { error: Error };
|
||||
expect(sent.error.message).toContain('postgres://svc:[redacted]@db.example.test/analytics');
|
||||
expect(sent.error.message).toContain('Authorization: [redacted]');
|
||||
expect(sent.error.message).not.toContain('db-url-secret');
|
||||
expect(sent.error.message).not.toContain('abc123');
|
||||
});
|
||||
|
||||
it('does not use process-global secret discovery when no snapshot is supplied', async () => {
|
||||
vi.stubEnv('KTX_FAKE_SECRET', 'plain-secret-without-pattern');
|
||||
const { io } = makeIo();
|
||||
|
||||
await reportException({
|
||||
error: new Error('plain-secret-without-pattern'),
|
||||
context: { source: 'uncaughtException', handled: false, fatal: true },
|
||||
io,
|
||||
packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' },
|
||||
});
|
||||
|
||||
const sent = captures[0] as { error: Error };
|
||||
expect(sent.error.message).toContain('plain-secret-without-pattern');
|
||||
});
|
||||
|
||||
it('dedupes the same Error instance between operation and global tiers', async () => {
|
||||
const { io } = makeIo();
|
||||
const error = new Error('same object');
|
||||
|
||||
await reportException({
|
||||
error,
|
||||
context: { source: 'scan run', handled: true, fatal: false },
|
||||
io,
|
||||
packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' },
|
||||
});
|
||||
await reportException({
|
||||
error,
|
||||
context: { source: 'uncaughtException', handled: false, fatal: true },
|
||||
io,
|
||||
packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' },
|
||||
immediate: true,
|
||||
});
|
||||
|
||||
expect(captures).toHaveLength(1);
|
||||
expect(immediateCaptures).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('captures wrapped Error causes as distinct logical occurrences', async () => {
|
||||
const { io } = makeIo();
|
||||
const inner = new Error('inner');
|
||||
const wrapper = new Error('outer', { cause: inner });
|
||||
|
||||
await reportException({
|
||||
error: inner,
|
||||
context: { source: 'sl query', handled: true, fatal: false },
|
||||
io,
|
||||
packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' },
|
||||
});
|
||||
await reportException({
|
||||
error: wrapper,
|
||||
context: { source: 'uncaughtException', handled: false, fatal: true },
|
||||
io,
|
||||
packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' },
|
||||
immediate: true,
|
||||
});
|
||||
|
||||
expect(captures).toHaveLength(1);
|
||||
expect(immediateCaptures).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('dedupes primitive and plain-object throwables propagated to the global tier', async () => {
|
||||
const { io } = makeIo();
|
||||
const objectThrowable = { message: 'plain object' };
|
||||
|
||||
await reportException({
|
||||
error: 'primitive boom',
|
||||
context: { source: 'mcp:sql_execution', handled: true, fatal: false },
|
||||
io,
|
||||
packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' },
|
||||
});
|
||||
await reportException({
|
||||
error: 'primitive boom',
|
||||
context: { source: 'unhandledRejection', handled: false, fatal: true },
|
||||
io,
|
||||
packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' },
|
||||
immediate: true,
|
||||
});
|
||||
await reportException({
|
||||
error: objectThrowable,
|
||||
context: { source: 'mcp:discover_data', handled: true, fatal: false },
|
||||
io,
|
||||
packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' },
|
||||
});
|
||||
await reportException({
|
||||
error: objectThrowable,
|
||||
context: { source: 'unhandledRejection', handled: false, fatal: true },
|
||||
io,
|
||||
packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' },
|
||||
immediate: true,
|
||||
});
|
||||
|
||||
expect(captures).toHaveLength(2);
|
||||
expect(immediateCaptures).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('does not collapse independent primitive throw events with the same value', async () => {
|
||||
const { io } = makeIo();
|
||||
|
||||
await reportException({
|
||||
error: 'oops',
|
||||
context: { source: 'scan run', handled: true, fatal: false },
|
||||
io,
|
||||
packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' },
|
||||
});
|
||||
await reportException({
|
||||
error: 'oops',
|
||||
context: { source: 'sql run', handled: true, fatal: false },
|
||||
io,
|
||||
packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' },
|
||||
});
|
||||
|
||||
expect(captures).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('drops forbidden caller-supplied extra property keys', async () => {
|
||||
const { io } = makeIo();
|
||||
|
||||
await reportException({
|
||||
error: new Error('extra property boom'),
|
||||
context: {
|
||||
source: 'sql run',
|
||||
handled: true,
|
||||
fatal: false,
|
||||
extra: {
|
||||
sql: 'select * from private_table',
|
||||
tableName: 'private_table',
|
||||
schemaName: 'private_schema',
|
||||
columnName: 'private_column',
|
||||
argv: '--password secret',
|
||||
env: 'KTX_TOKEN=secret',
|
||||
password: 'secret-password', // pragma: allowlist secret
|
||||
token: 'secret-token',
|
||||
prompt: 'user prompt',
|
||||
safeCount: 3,
|
||||
},
|
||||
},
|
||||
io,
|
||||
packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' },
|
||||
});
|
||||
|
||||
const sent = captures[0] as { properties: Record<string, unknown> };
|
||||
expect(sent.properties.safeCount).toBe(3);
|
||||
for (const key of [
|
||||
'sql',
|
||||
'tableName',
|
||||
'schemaName',
|
||||
'columnName',
|
||||
'argv',
|
||||
'env',
|
||||
'password',
|
||||
'token',
|
||||
'prompt',
|
||||
]) {
|
||||
expect(sent.properties).not.toHaveProperty(key);
|
||||
}
|
||||
});
|
||||
|
||||
it('redacts every required static credential pattern and leaves benign text intact', async () => {
|
||||
const { io } = makeIo();
|
||||
const cases: Array<{ message: string; leaked: string; expected: string }> = [
|
||||
{
|
||||
message: 'dsn password=hunter2',
|
||||
leaked: 'hunter2',
|
||||
expected: 'password=[redacted]',
|
||||
},
|
||||
{
|
||||
message: 'dsn pwd=swordfish',
|
||||
leaked: 'swordfish',
|
||||
expected: 'pwd=[redacted]',
|
||||
},
|
||||
{
|
||||
message: 'Authorization: Basic abc123',
|
||||
leaked: 'abc123',
|
||||
expected: 'Authorization: [redacted]',
|
||||
},
|
||||
{
|
||||
message: 'Authorization: Bearer token-123',
|
||||
leaked: 'token-123',
|
||||
expected: 'Authorization: [redacted]',
|
||||
},
|
||||
{
|
||||
message: 'Bearer standalone-token',
|
||||
leaked: 'standalone-token',
|
||||
expected: 'Bearer [redacted]',
|
||||
},
|
||||
{
|
||||
message: 'api_key=sk-live-secret',
|
||||
leaked: 'sk-live-secret',
|
||||
expected: 'api_key=[redacted]',
|
||||
},
|
||||
{
|
||||
message: 'api-key: sk-dash-secret',
|
||||
leaked: 'sk-dash-secret',
|
||||
expected: 'api-key=[redacted]',
|
||||
},
|
||||
{
|
||||
message: 'KTX_PROVIDER_TOKEN=ktx-secret',
|
||||
leaked: 'ktx-secret',
|
||||
expected: 'KTX_PROVIDER_TOKEN=[redacted]',
|
||||
},
|
||||
{
|
||||
message: 'REFRESH_SECRET: refresh-secret',
|
||||
leaked: 'refresh-secret',
|
||||
expected: 'REFRESH_SECRET=[redacted]',
|
||||
},
|
||||
{
|
||||
message: 'https://s3.example.test/file?X-Amz-Signature=aws-secret&ok=1',
|
||||
leaked: 'aws-secret',
|
||||
expected: 'X-Amz-Signature=[redacted]',
|
||||
},
|
||||
{
|
||||
message: 'https://storage.example.test/file?X-Goog-Signature=goog-secret&ok=1',
|
||||
leaked: 'goog-secret',
|
||||
expected: 'X-Goog-Signature=[redacted]',
|
||||
},
|
||||
{
|
||||
message: 'https://cdn.example.test/file?sig=signed-secret&ok=1',
|
||||
leaked: 'signed-secret',
|
||||
expected: 'sig=[redacted]',
|
||||
},
|
||||
{
|
||||
message: 'postgres://svc:url-password@db.example.test/analytics', // pragma: allowlist secret
|
||||
leaked: 'url-password',
|
||||
expected: 'postgres://svc:[redacted]@db.example.test/analytics',
|
||||
},
|
||||
];
|
||||
|
||||
for (const item of cases) {
|
||||
await reportException({
|
||||
error: new Error(item.message),
|
||||
context: { source: 'connection test', handled: true, fatal: false },
|
||||
io,
|
||||
packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' },
|
||||
});
|
||||
const sent = captures[captures.length - 1] as { error: Error };
|
||||
expect(sent.error.message).toContain(item.expected);
|
||||
expect(sent.error.message).not.toContain(item.leaked);
|
||||
}
|
||||
|
||||
await reportException({
|
||||
error: new Error('token bucket metrics and passwordless auth are benign'),
|
||||
context: { source: 'connection test', handled: true, fatal: false },
|
||||
io,
|
||||
packageInfo: { name: '@kaelio/ktx', version: '0.0.0-test' },
|
||||
});
|
||||
const benign = captures[captures.length - 1] as { error: Error };
|
||||
expect(benign.error.message).toBe('token bucket metrics and passwordless auth are benign');
|
||||
});
|
||||
});
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue