mirror of
https://github.com/Kaelio/ktx.git
synced 2026-06-07 07:55:13 +02:00
Compare commits
No commits in common. "main" and "v0.7.0" have entirely different histories.
284 changed files with 6292 additions and 24216 deletions
4
.github/workflows/ci.yml
vendored
4
.github/workflows/ci.yml
vendored
|
|
@ -217,7 +217,7 @@ jobs:
|
|||
flags: typescript
|
||||
name: typescript
|
||||
disable_search: true
|
||||
fail_ci_if_error: false
|
||||
fail_ci_if_error: true
|
||||
|
||||
- name: Warn when Codecov token is missing for TypeScript
|
||||
if: env.CODECOV_TOKEN_CONFIGURED != 'true'
|
||||
|
|
@ -236,7 +236,7 @@ jobs:
|
|||
flags: python
|
||||
name: python
|
||||
disable_search: true
|
||||
fail_ci_if_error: false
|
||||
fail_ci_if_error: true
|
||||
|
||||
- name: Warn when Codecov token is missing for Python
|
||||
if: env.CODECOV_TOKEN_CONFIGURED != 'true'
|
||||
|
|
|
|||
72
.github/workflows/star-history.yml
vendored
72
.github/workflows/star-history.yml
vendored
|
|
@ -1,72 +0,0 @@
|
|||
name: Refresh star history chart
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Twice daily at 06:00 and 18:00 UTC.
|
||||
- cron: "0 6,18 * * *"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
env:
|
||||
DO_NOT_TRACK: "1"
|
||||
KTX_TELEMETRY_DISABLED: "1"
|
||||
NEXT_TELEMETRY_DISABLED: "1"
|
||||
|
||||
concurrency:
|
||||
group: star-history-refresh
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
refresh:
|
||||
name: Regenerate assets/star-history.svg
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
# RELEASE_PAT can push to the protected main branch; the default
|
||||
# GITHUB_TOKEN is rejected by the branch-protection hook (GH006).
|
||||
token: ${{ secrets.RELEASE_PAT }}
|
||||
|
||||
- name: Fetch fresh star-history SVG
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# cachebust forces star-history to regenerate instead of serving its
|
||||
# own server-side cache; --location follows the slug-normalizing 301.
|
||||
url="https://api.star-history.com/svg?repos=Kaelio/ktx&type=Date&cachebust=${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
curl --fail --location --silent --show-error \
|
||||
--retry 3 --retry-delay 5 --max-time 60 \
|
||||
-o assets/star-history.svg.new "$url"
|
||||
# Guard against error pages / truncated responses before overwriting.
|
||||
if ! grep -q "</svg>" assets/star-history.svg.new; then
|
||||
echo "Downloaded file is not a valid SVG; aborting." >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ "$(wc -c < assets/star-history.svg.new)" -lt 1000 ]; then
|
||||
echo "Downloaded SVG is suspiciously small; aborting." >&2
|
||||
exit 1
|
||||
fi
|
||||
# The star-history API returns the SVG without a trailing newline,
|
||||
# which end-of-file-fixer rewrites whenever pre-commit runs
|
||||
# --all-files on a PR. Because the refresh commit below uses [skip ci],
|
||||
# the hook never runs against it here, so an un-normalized file
|
||||
# silently breaks the pre-commit check on every open PR. Normalize to
|
||||
# exactly one trailing newline before committing.
|
||||
printf '%s\n' "$(cat assets/star-history.svg.new)" > assets/star-history.svg
|
||||
rm -f assets/star-history.svg.new
|
||||
|
||||
- name: Commit if changed
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if git diff --quiet -- assets/star-history.svg; then
|
||||
echo "Star-history chart unchanged; nothing to commit."
|
||||
exit 0
|
||||
fi
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git add assets/star-history.svg
|
||||
# [skip ci] keeps this housekeeping commit from triggering KTX CI.
|
||||
git commit -m "chore: refresh star history chart [skip ci]"
|
||||
git push
|
||||
2
.github/workflows/triage-issues.yml
vendored
2
.github/workflows/triage-issues.yml
vendored
|
|
@ -22,7 +22,7 @@ jobs:
|
|||
github.event.issue.author_association != 'COLLABORATOR'
|
||||
steps:
|
||||
- name: Apply needs-triage label
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
script: |
|
||||
await github.rest.issues.addLabels({
|
||||
|
|
|
|||
|
|
@ -14,18 +14,6 @@ repos:
|
|||
- id: check-case-conflict
|
||||
- id: mixed-line-ending
|
||||
|
||||
- repo: https://github.com/tombi-toml/tombi-pre-commit
|
||||
rev: v1.1.0
|
||||
hooks:
|
||||
- id: tombi-format
|
||||
args: ["--offline"]
|
||||
# uv.lock is generated and owned by uv, which writes its own canonical
|
||||
# TOML layout. tombi reformats that layout differently, so once uv
|
||||
# regenerates the lock (e.g. after a dependency or version change)
|
||||
# tombi rewrites it and the hook fails on the modified file. Keep uv
|
||||
# authoritative for its lockfile; tombi still formats hand-edited TOML.
|
||||
exclude: ^uv\.lock$
|
||||
|
||||
- repo: https://github.com/asottile/pyupgrade
|
||||
rev: v3.21.2
|
||||
hooks:
|
||||
|
|
|
|||
88
AGENTS.md
88
AGENTS.md
|
|
@ -159,65 +159,6 @@ and naming asymmetries are bugs in waiting — see
|
|||
[`docs/code-design.md`](docs/code-design.md). Treat the `MUST` / `MUST NOT`
|
||||
rules there with the same weight as the ones in this file.
|
||||
|
||||
## Design Reasoning Defaults
|
||||
|
||||
When proposing a design, an approach, or any non-trivial change, apply these
|
||||
defaults and run the self-check before presenting it. They encode the
|
||||
corrections users most often have to make; reaching these conclusions
|
||||
autonomously — without being asked the leading question — is the bar.
|
||||
|
||||
- **MUST**: Optimize for the best outcome, not for an unstated constraint. Do not
|
||||
silently adopt "smallest change", "least effort", "cheapest", or "least user
|
||||
intervention" as the goal unless the user said so. Default to the most correct,
|
||||
durable solution, and present cost / effort / scope as information for the user
|
||||
to weigh — not as a ceiling you impose on their behalf.
|
||||
- **MUST**: Separate one-time cost from recurring cost before discarding an
|
||||
option. A fixed cost paid once (a setup-time computation, an extra LLM call
|
||||
during setup, a contract change) to make every later run cheaper or more
|
||||
correct is usually worth it. Do not reject it with recurring-cost reasoning;
|
||||
quantify both sides. (Example smell: "don't add an LLM call to a cost-cutting
|
||||
feature" — wrong when the call is one-time and the savings recur.)
|
||||
- **MUST**: Treat a user's example as a representative of a class, not as the
|
||||
spec. Design for the general population the example stands for, then stress-test
|
||||
against deliberately different instances — another warehouse, dialect, stack
|
||||
layout, or input shape — before committing. If a design only works because of an
|
||||
incidental property of the example (e.g. "the noise happened to be in a separate
|
||||
schema *on this demo*"), it is curve-fitting; generalize it or state the
|
||||
assumption explicitly.
|
||||
- **MUST**: Prefer deriving from the system's own state over enumerating cases.
|
||||
Favor an allowlist computed from declared/observed state (config, scanned
|
||||
catalog, query log, the user's own inputs) over a denylist of known-bad
|
||||
specifics (particular tables, schemas, tools, or vendors). A hardcoded or
|
||||
hand-maintained list of external specifics is a smell: it rots and fails on the
|
||||
next stack. The only acceptable static patterns are genuinely universal
|
||||
invariants (e.g. DB-engine system catalogs) and ktx's own self-emitted
|
||||
signatures.
|
||||
- **SHOULD**: Before inventing an abstraction or hand-rolling structural logic,
|
||||
search for what already exists and reuse it — the codebase's canonical
|
||||
representation (a structured ref/key type) instead of a parallel string scheme,
|
||||
and a mandated/available tool (e.g. `sqlglot` for SQL structure; see
|
||||
[SQL and Structured Parsing](#sql-and-structured-parsing)) instead of
|
||||
hand-parsing. Normalize ambiguous input to the canonical form at the boundary;
|
||||
do not carry the ambiguity downstream. This is the single-source-of-truth / DRY
|
||||
item from the Priority Hierarchy applied at design time.
|
||||
|
||||
Before presenting a design, answer these explicitly:
|
||||
|
||||
1. Am I optimizing for a goal the user actually stated, or one I assumed?
|
||||
2. Does this generalize beyond the example in front of me? Name a real case where
|
||||
it would break.
|
||||
3. Am I enumerating known-bad cases when I could derive scope from the system's
|
||||
own declared/observed state?
|
||||
4. Is there an existing canonical representation or mandated tool I should reuse
|
||||
instead of building or parsing my own?
|
||||
5. Am I discarding the better option on a weak or misapplied constraint
|
||||
(one-time vs recurring cost, "more surface area", "more work now")?
|
||||
|
||||
A user question that nudges toward an alternative ("would X help?", "should I
|
||||
always do Y?", "will you hardcode Z?") is a signal that a better option exists.
|
||||
Investigate the implied direction and reason it through *before* defending the
|
||||
original proposal — and prefer to have asked yourself the question first.
|
||||
|
||||
## TypeScript Standards
|
||||
|
||||
- Use Node 22+ and pnpm workspace commands.
|
||||
|
|
@ -337,8 +278,7 @@ use `PascalCase` without the suffix.
|
|||
|
||||
## Telemetry
|
||||
|
||||
**ktx** ships PostHog usage telemetry. Catalog telemetry events use strict
|
||||
schemas. When adding commands or events:
|
||||
**ktx** ships anonymous PostHog telemetry. When adding commands or events:
|
||||
|
||||
- **MUST NOT**: Add fields that carry user data — file paths, hostnames,
|
||||
environment values, SQL text, schema/table/column names, error messages,
|
||||
|
|
@ -355,24 +295,6 @@ schemas. When adding commands or events:
|
|||
of collected data changes. Adding another event with no new field types
|
||||
needs no docs change.
|
||||
|
||||
### Error reports
|
||||
|
||||
**ktx** also sends PostHog Error Tracking `$exception` events when telemetry is
|
||||
enabled. This channel is separate from the strict catalog event schema and is
|
||||
used only for exception diagnostics.
|
||||
|
||||
`$exception` events may include stack frames, error class names, raw error
|
||||
messages, cause chains, `source`, `handled`, `fatal`, runtime version fields,
|
||||
OS/runtime fields, and the hashed `projectId` when known. Stack frames may
|
||||
include local file paths and the local username when those appear in paths.
|
||||
|
||||
`$exception` events must never intentionally include secrets, credentials,
|
||||
database URLs, auth headers, raw argv, raw environment values, SQL text,
|
||||
schema/table/column names as explicit properties, customer row data, user prompt
|
||||
text, or raw MCP arguments. Reporters must redact call-site-provided secret
|
||||
snapshots and common static credential patterns before the SDK serializes the
|
||||
exception.
|
||||
|
||||
## Documentation and Specs
|
||||
|
||||
- Keep public documentation in `README.md`, package READMEs, example READMEs,
|
||||
|
|
@ -428,9 +350,8 @@ error messages — including the disambiguation rule for the overloaded word
|
|||
`source` (semantic / primary / context / source of truth) — see
|
||||
[`docs/terminology.md`](docs/terminology.md). Follow that file when choosing
|
||||
between near-synonyms (e.g. `connector` vs `adapter`, `data agent` vs
|
||||
`database agent`, `context-source ingest` vs `source ingest`). Product-name
|
||||
rules in this section take precedence over anything in that file when they
|
||||
conflict.
|
||||
`database agent`, `fast ingest` vs `schema ingest`). Product-name rules in
|
||||
this section take precedence over anything in that file when they conflict.
|
||||
|
||||
### Updating `docs-site/` After Code Changes
|
||||
|
||||
|
|
@ -459,8 +380,7 @@ rather than silently skipping it.
|
|||
- **MUST**: Disable monospace ligatures on every surface that uses the
|
||||
`var(--font-mono)` family (Geist Mono). Geist Mono fuses `--` into an
|
||||
em-dash glyph that visually eats the adjacent space, so prompts like
|
||||
`npx skills add Kaelio/ktx --skill ktx` render as
|
||||
`Kaelio/ktx--skill ktx`.
|
||||
`npx skills add Kaelio/ktx --skill ktx` render as `Kaelio/ktx--skill ktx`.
|
||||
- **MUST**: When adding a new container that renders user-visible monospace
|
||||
text outside `<code>` / `<pre>` (e.g. a styled `<div className="font-mono">`
|
||||
for a copyable prompt), verify the global ligature-off rule in
|
||||
|
|
|
|||
47
README.md
47
README.md
|
|
@ -13,7 +13,7 @@
|
|||
<a href="https://docs.kaelio.com/ktx/docs/"><img src="https://img.shields.io/badge/docs-ktx-22c55e?style=flat-square" alt="Documentation" /></a>
|
||||
<a href="https://join.slack.com/t/ktxcommunity/shared_invite/zt-3y9b44m1x-LVyNNJD5nwaZHq4XS29LMQ"><img src="https://img.shields.io/badge/slack-join%20community-4A154B?style=flat-square&logo=slack&logoColor=white" alt="Join the ktx Slack community" /></a>
|
||||
<a href="https://github.com/Kaelio/ktx/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue?style=flat-square" alt="License" /></a>
|
||||
<a href="https://www.ycombinator.com/companies/kaelio"><img src="https://img.shields.io/badge/Y%20Combinator-P25-orange?style=flat-square" alt="Y Combinator P25" /></a>
|
||||
<a href="https://www.ycombinator.com/companies?batch=P25"><img src="https://img.shields.io/badge/Y%20Combinator-P25-orange?style=flat-square" alt="Y Combinator P25" /></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
|
|
@ -23,10 +23,6 @@
|
|||
<a href="https://join.slack.com/t/ktxcommunity/shared_invite/zt-3y9b44m1x-LVyNNJD5nwaZHq4XS29LMQ"><b>Slack</b></a>
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<sub>Built and maintained by <a href="https://www.kaelio.com"><b>Kaelio</b></a></sub>
|
||||
</p>
|
||||
|
||||
---
|
||||
|
||||
**ktx** is a self-improving context layer that teaches agents how to query your
|
||||
|
|
@ -34,25 +30,13 @@ warehouse accurately - from approved metric definitions, joinable columns, and
|
|||
business knowledge it builds and maintains for you.
|
||||
|
||||
> [!NOTE]
|
||||
> Run **ktx** with your own LLM API keys or a local agent sign-in — a
|
||||
> **Claude Pro/Max** subscription through Claude Code, or your local Codex
|
||||
> authentication. No extra usage billing from **ktx**.
|
||||
> Run **ktx** with your own LLM API keys or a **Claude Pro/Max** subscription.
|
||||
> No extra usage billing from **ktx**.
|
||||
|
||||
<p align="center">
|
||||
<a href="https://youtu.be/5V4TuzYVlrA">
|
||||
<img src="assets/launch-video-thumb.png" alt="Watch the ktx launch video (1:56)" width="820" />
|
||||
</a>
|
||||
<img src="docs-site/public/images/ingestion-flow-transparent.svg" alt="ktx ingestion flow from source systems through validation to wiki and semantic-layer outputs" width="900" />
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<img src="docs-site/public/images/ingestion-flow.png" alt="Ingestion: ktx ingests databases, BI tools, modeling code, and docs through its context engine (source connectors, context builder, reconciliation, validation) into wiki Markdown and semantic-layer YAML" width="900" />
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<img src="docs-site/public/images/mcp-runtime-flow.png" alt="Serving: an agent queries ktx through MCP, which searches the wiki and semantic layer, returns approved metrics, and compiles them into read-only SQL run against the warehouse" width="900" />
|
||||
</p>
|
||||
|
||||
|
||||
## Why ktx
|
||||
|
||||
General-purpose agents struggle on data tasks. They re-explore your warehouse
|
||||
|
|
@ -180,9 +164,8 @@ then the current directory. Pass `--project-dir <path>` when scripting.
|
|||
No. **ktx** runs locally. The only data leaving your machine is what you
|
||||
send to the LLM provider you configured.
|
||||
- **Which LLM backends are supported?**
|
||||
Anthropic API, Google Vertex AI, AI Gateway, the local Claude Code session
|
||||
through the Claude Agent SDK, and your local Codex authentication through the
|
||||
Codex SDK. See
|
||||
Anthropic API, Google Vertex AI, AI Gateway, and the local Claude Code
|
||||
session through the Claude Agent SDK. See
|
||||
[LLM configuration](https://docs.kaelio.com/ktx/docs/guides/llm-configuration).
|
||||
- **How is ktx different from a dbt or MetricFlow semantic layer?**
|
||||
**ktx** *ingests* those layers and combines them with raw-table
|
||||
|
|
@ -251,17 +234,11 @@ uv run pytest -q
|
|||
|
||||
## Telemetry
|
||||
|
||||
**ktx** collects privacy-conscious usage telemetry to understand installs and
|
||||
improve setup, command reliability, and data-agent workflows. Catalog telemetry
|
||||
events do not record file paths, hostnames, SQL, schema names, table names,
|
||||
column names, error messages, raw environment values, or argv. Error reports use
|
||||
PostHog Error Tracking and can include stack frames and raw error messages,
|
||||
which may contain local file paths or the local username in those paths.
|
||||
**ktx** redacts secrets, credentials, database URLs, auth headers, argv, raw
|
||||
environment values, SQL text, row data, and user-typed prompt or MCP argument
|
||||
text from the explicit `$exception` payload. See
|
||||
[Telemetry](https://docs.kaelio.com/ktx/docs/community/telemetry) for the event
|
||||
catalog and opt-out options.
|
||||
**ktx** collects anonymous usage telemetry from interactive CLI runs to
|
||||
improve setup, command reliability, and data-agent workflows. No file paths,
|
||||
hostnames, SQL, schema names, error messages, or argv are recorded. See
|
||||
[Telemetry](https://docs.kaelio.com/ktx/docs/community/telemetry) for the
|
||||
event catalog and opt-out options.
|
||||
|
||||
## License
|
||||
|
||||
|
|
@ -271,6 +248,6 @@ catalog and opt-out options.
|
|||
|
||||
<p align="center">
|
||||
<a href="https://star-history.com/#Kaelio/ktx&Date">
|
||||
<img src="assets/star-history.svg" alt="ktx Star History Chart" width="700" />
|
||||
<img src="https://api.star-history.com/svg?repos=Kaelio/ktx&type=Date" alt="ktx Star History Chart" width="700" />
|
||||
</a>
|
||||
</p>
|
||||
|
|
|
|||
|
|
@ -19,9 +19,14 @@
|
|||
<path d="M 80 84 Q 86 77 92 84" fill="none" stroke="#F5F1EA" stroke-width="3.5" stroke-linecap="round" />
|
||||
<path d="M 108 84 Q 114 77 120 84" fill="none" stroke="#F5F1EA" stroke-width="3.5" stroke-linecap="round" />
|
||||
|
||||
<!-- wordmark: "ktx" outlined from Outfit SemiBold (the docs-site display font)
|
||||
so it renders identically everywhere, independent of installed fonts -->
|
||||
<g transform="translate(242 145)" fill="#1B3139">
|
||||
<path d="M51.17 0 25.06 -34.79 51.03 -67.62H72.17L41.65 -30.7L42.35 -39.55L73.57 0ZM8.05 0V-101.22H26.46V0ZM88.41 0V-95.69H106.82V0ZM72.66 -51.52V-67.62H122.57V-51.52ZM171.75 0 153.93 -27.41 150.22 -30.17 123.83 -67.62H145.64L161.91 -42.77L165.48 -40.18L193.38 0ZM122.54 0 150.05 -38.61 160.62 -26.22 143.19 0ZM166.11 -30.38 155.44 -42.67 171.54 -67.62H192.08Z" />
|
||||
</g>
|
||||
<!-- wordmark: 'ktx', half the logo height, vertically centered -->
|
||||
<text
|
||||
x="225"
|
||||
y="145"
|
||||
font-family="'JetBrains Mono', 'Fira Code', ui-monospace, 'SF Mono', Menlo, monospace"
|
||||
font-size="140"
|
||||
font-weight="600"
|
||||
fill="#1B3139"
|
||||
letter-spacing="-0.04em"
|
||||
>ktx</text>
|
||||
</svg>
|
||||
|
|
|
|||
|
Before Width: | Height: | Size: 1.4 KiB After Width: | Height: | Size: 1.1 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 135 KiB |
File diff suppressed because one or more lines are too long
|
Before Width: | Height: | Size: 63 KiB |
|
|
@ -1,12 +0,0 @@
|
|||
import type { Metadata } from "next";
|
||||
|
||||
import { DiagramStudio } from "@/components/diagram-studio/studio";
|
||||
|
||||
export const metadata: Metadata = {
|
||||
title: "Diagram studio",
|
||||
robots: { index: false, follow: false },
|
||||
};
|
||||
|
||||
export default function DiagramStudioPage() {
|
||||
return <DiagramStudio />;
|
||||
}
|
||||
|
|
@ -5,7 +5,7 @@ import { SlackIcon } from "@/components/slack-icon";
|
|||
|
||||
export const baseOptions: BaseLayoutProps = {
|
||||
nav: {
|
||||
title: Logo,
|
||||
title: <Logo />,
|
||||
transparentMode: "top",
|
||||
},
|
||||
links: [
|
||||
|
|
|
|||
|
|
@ -1,328 +0,0 @@
|
|||
import { type Edge, MarkerType, type Node } from "@xyflow/react";
|
||||
|
||||
import { C } from "./nodes";
|
||||
|
||||
const EDGE_COLOR = "#b3bcc4";
|
||||
const MARKER_COLOR = "#9aa6ad";
|
||||
|
||||
const labelStyle = {
|
||||
fontFamily: "var(--font-inter), system-ui, sans-serif",
|
||||
fontSize: 15,
|
||||
fontWeight: 600,
|
||||
fill: C.inkMuted,
|
||||
};
|
||||
const labelBgStyle = { fill: "#ffffff", stroke: C.chipBorder, strokeWidth: 1 };
|
||||
const labelBg = {
|
||||
labelBgPadding: [8, 4] as [number, number],
|
||||
labelBgBorderRadius: 6,
|
||||
labelStyle,
|
||||
labelBgStyle,
|
||||
};
|
||||
|
||||
const marker = { type: MarkerType.ArrowClosed, color: MARKER_COLOR, width: 16, height: 16 };
|
||||
const edgeStyle = { stroke: EDGE_COLOR, strokeWidth: 2 };
|
||||
|
||||
/* ============================== INGESTION =============================== */
|
||||
|
||||
const SRC_W = 300;
|
||||
const SRC_H = 138;
|
||||
const SRC_GAP = 24;
|
||||
const srcY = (i: number) => i * (SRC_H + SRC_GAP);
|
||||
|
||||
export const ingestionNodes: Node[] = [
|
||||
{
|
||||
id: "title",
|
||||
type: "title",
|
||||
position: { x: 0, y: -96 },
|
||||
data: {
|
||||
width: 560,
|
||||
eyebrow: "1 · Ingestion",
|
||||
title: "ktx builds your context layer",
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "db",
|
||||
type: "card",
|
||||
position: { x: 0, y: srcY(0) },
|
||||
data: {
|
||||
width: SRC_W,
|
||||
height: SRC_H,
|
||||
accent: C.teal,
|
||||
rows: [
|
||||
{ kind: "title", text: "Databases" },
|
||||
{ kind: "desc", text: "Schemas, keys, query history" },
|
||||
{ kind: "muted", text: "Postgres · Snowflake · BigQuery · …" },
|
||||
],
|
||||
handles: [{ side: "right", type: "source", id: "out" }],
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "bi",
|
||||
type: "card",
|
||||
position: { x: 0, y: srcY(1) },
|
||||
data: {
|
||||
width: SRC_W,
|
||||
height: SRC_H,
|
||||
accent: C.orange,
|
||||
rows: [
|
||||
{ kind: "title", text: "BI tools" },
|
||||
{ kind: "desc", text: "Dashboards, explores, usage" },
|
||||
{ kind: "muted", text: "Metabase · Looker · …" },
|
||||
],
|
||||
handles: [{ side: "right", type: "source", id: "out" }],
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "model",
|
||||
type: "card",
|
||||
position: { x: 0, y: srcY(2) },
|
||||
data: {
|
||||
width: SRC_W,
|
||||
height: SRC_H,
|
||||
accent: C.amber,
|
||||
rows: [
|
||||
{ kind: "title", text: "Modeling code" },
|
||||
{ kind: "desc", text: "Metrics, models, joins, entities" },
|
||||
{ kind: "muted", text: "dbt · LookML · MetricFlow · …" },
|
||||
],
|
||||
handles: [{ side: "right", type: "source", id: "out" }],
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "docs",
|
||||
type: "card",
|
||||
position: { x: 0, y: srcY(3) },
|
||||
data: {
|
||||
width: SRC_W,
|
||||
height: SRC_H,
|
||||
accent: C.emerald,
|
||||
rows: [
|
||||
{ kind: "title", text: "Docs & notes" },
|
||||
{ kind: "desc", text: "Policies, definitions, notes" },
|
||||
{ kind: "muted", text: "Notion · any text · …" },
|
||||
],
|
||||
handles: [{ side: "right", type: "source", id: "out" }],
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "engine",
|
||||
type: "engine",
|
||||
position: { x: 420, y: 52 },
|
||||
data: {
|
||||
width: 380,
|
||||
height: 520,
|
||||
steps: [
|
||||
{ n: 1, title: "Source connectors", desc: "Read each source in its shape" },
|
||||
{ n: 2, title: "Context builder", desc: "Evidence into proposed updates" },
|
||||
{ n: 3, title: "Reconciliation", desc: "Merge with existing context" },
|
||||
{ n: 4, title: "Validation", desc: "Check references & semantics" },
|
||||
],
|
||||
handles: [
|
||||
{ side: "left", type: "target", id: "in" },
|
||||
{ side: "right", type: "source", id: "out" },
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "wiki",
|
||||
type: "card",
|
||||
position: { x: 900, y: 66 },
|
||||
data: {
|
||||
width: 320,
|
||||
height: 220,
|
||||
accent: C.emerald,
|
||||
rows: [
|
||||
{ kind: "mono", text: "wiki/*.md", color: C.emerald },
|
||||
{ kind: "title", text: "Wiki" },
|
||||
{ kind: "chips", items: ["free-form", "auto-maintained"] },
|
||||
{ kind: "desc", text: "Definitions, caveats, policies," },
|
||||
{ kind: "desc", text: "and notes agents can search." },
|
||||
],
|
||||
handles: [{ side: "left", type: "target", id: "in" }],
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "sl",
|
||||
type: "card",
|
||||
position: { x: 900, y: 338 },
|
||||
data: {
|
||||
width: 320,
|
||||
height: 220,
|
||||
accent: C.teal,
|
||||
rows: [
|
||||
{ kind: "mono", text: "semantic-layer/*.yaml", color: C.teal },
|
||||
{ kind: "title", text: "Semantic layer" },
|
||||
{ kind: "chips", items: ["executable", "auto-maintained"] },
|
||||
{ kind: "desc", text: "Metrics, joins, dimensions, and" },
|
||||
{ kind: "desc", text: "filters ktx compiles into SQL." },
|
||||
],
|
||||
handles: [{ side: "left", type: "target", id: "in" }],
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const ingestEdge = (source: string, target: string): Edge => ({
|
||||
id: `${source}-${target}`,
|
||||
source,
|
||||
target,
|
||||
sourceHandle: "out",
|
||||
targetHandle: "in",
|
||||
type: "default",
|
||||
style: edgeStyle,
|
||||
markerEnd: marker,
|
||||
});
|
||||
|
||||
export const ingestionEdges: Edge[] = [
|
||||
ingestEdge("db", "engine"),
|
||||
ingestEdge("bi", "engine"),
|
||||
ingestEdge("model", "engine"),
|
||||
ingestEdge("docs", "engine"),
|
||||
ingestEdge("engine", "wiki"),
|
||||
ingestEdge("engine", "sl"),
|
||||
];
|
||||
|
||||
/* =============================== RUNTIME ================================ */
|
||||
|
||||
export const runtimeNodes: Node[] = [
|
||||
{
|
||||
id: "title",
|
||||
type: "title",
|
||||
position: { x: 0, y: -84 },
|
||||
data: {
|
||||
width: 560,
|
||||
eyebrow: "2 · Serving",
|
||||
title: "agents query it through MCP",
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "agent",
|
||||
type: "card",
|
||||
position: { x: 0, y: 115 },
|
||||
data: {
|
||||
width: 280,
|
||||
height: 190,
|
||||
accent: C.neutral,
|
||||
align: "center",
|
||||
rows: [
|
||||
{ kind: "title", text: "Your agent" },
|
||||
{ kind: "muted", text: "Claude Code · Cursor" },
|
||||
{ kind: "muted", text: "Codex · OpenCode" },
|
||||
],
|
||||
handles: [
|
||||
{ side: "right", type: "source", id: "ask", top: "42%" },
|
||||
{ side: "right", type: "target", id: "answer", top: "62%" },
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "hub",
|
||||
type: "hub",
|
||||
position: { x: 420, y: 85 },
|
||||
data: {
|
||||
width: 360,
|
||||
height: 250,
|
||||
rows: [
|
||||
"Search wiki + semantic layer",
|
||||
"Return approved metrics",
|
||||
"Compile metrics → SQL",
|
||||
],
|
||||
handles: [
|
||||
{ side: "left", type: "target", id: "ask", top: "42%" },
|
||||
{ side: "left", type: "source", id: "answer", top: "62%" },
|
||||
{ side: "right", type: "source", id: "to-context", top: "30%" },
|
||||
{ side: "right", type: "source", id: "to-warehouse", top: "72%" },
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "context",
|
||||
type: "card",
|
||||
position: { x: 920, y: 15 },
|
||||
data: {
|
||||
width: 300,
|
||||
height: 150,
|
||||
accent: C.teal,
|
||||
rows: [
|
||||
{ kind: "title", text: "Context layer" },
|
||||
{ kind: "mono", text: "wiki/*.md", color: C.emerald },
|
||||
{ kind: "mono", text: "semantic-layer/*.yaml", color: C.teal },
|
||||
],
|
||||
handles: [{ side: "left", type: "target", id: "in" }],
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "warehouse",
|
||||
type: "card",
|
||||
position: { x: 920, y: 255 },
|
||||
data: {
|
||||
width: 300,
|
||||
height: 150,
|
||||
accent: C.slate,
|
||||
rows: [
|
||||
{ kind: "title", text: "Warehouse" },
|
||||
{
|
||||
kind: "badge",
|
||||
text: "read-only",
|
||||
bg: "#ecf6f8",
|
||||
border: "#bfe3ea",
|
||||
color: C.teal,
|
||||
},
|
||||
{ kind: "desc", text: "Runs the compiled SQL" },
|
||||
],
|
||||
handles: [{ side: "left", type: "target", id: "in" }],
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
export const runtimeEdges: Edge[] = [
|
||||
{
|
||||
id: "ask",
|
||||
source: "agent",
|
||||
sourceHandle: "ask",
|
||||
target: "hub",
|
||||
targetHandle: "ask",
|
||||
type: "default",
|
||||
label: "ask",
|
||||
...labelBg,
|
||||
style: edgeStyle,
|
||||
markerEnd: marker,
|
||||
},
|
||||
{
|
||||
id: "answer",
|
||||
source: "hub",
|
||||
sourceHandle: "answer",
|
||||
target: "agent",
|
||||
targetHandle: "answer",
|
||||
type: "default",
|
||||
label: "answer",
|
||||
...labelBg,
|
||||
style: edgeStyle,
|
||||
markerEnd: marker,
|
||||
},
|
||||
{
|
||||
id: "search",
|
||||
source: "hub",
|
||||
sourceHandle: "to-context",
|
||||
target: "context",
|
||||
targetHandle: "in",
|
||||
type: "smoothstep",
|
||||
label: "search + read",
|
||||
...labelBg,
|
||||
style: edgeStyle,
|
||||
markerStart: marker,
|
||||
markerEnd: marker,
|
||||
},
|
||||
{
|
||||
id: "readonly",
|
||||
source: "hub",
|
||||
sourceHandle: "to-warehouse",
|
||||
target: "warehouse",
|
||||
targetHandle: "in",
|
||||
type: "smoothstep",
|
||||
label: "read-only",
|
||||
...labelBg,
|
||||
style: edgeStyle,
|
||||
markerStart: marker,
|
||||
markerEnd: marker,
|
||||
},
|
||||
];
|
||||
|
|
@ -1,57 +0,0 @@
|
|||
/**
|
||||
* Inlined ktx mascot, ported from assets/ktx-mascot.svg.
|
||||
*
|
||||
* - `light` renders the dark-bodied mascot for light surfaces.
|
||||
* - `dark` renders the cream-bodied mascot for dark surfaces (e.g. the ktx
|
||||
* hub panel), mirroring brand/ktx-mascot-dark.svg.
|
||||
*/
|
||||
export function KtxMascot({
|
||||
variant = "light",
|
||||
size = 56,
|
||||
}: {
|
||||
variant?: "light" | "dark";
|
||||
size?: number;
|
||||
}) {
|
||||
const body = variant === "dark" ? "#F5F1EA" : "#1B3139";
|
||||
const eye = variant === "dark" ? "#1B3139" : "#F5F1EA";
|
||||
return (
|
||||
<svg
|
||||
viewBox="0 0 200 200"
|
||||
width={size}
|
||||
height={size}
|
||||
role="img"
|
||||
aria-label="ktx mascot"
|
||||
>
|
||||
<g fill="none" stroke={body} strokeWidth="16" strokeLinecap="round">
|
||||
<path d="M 62 110 Q 32 130 44 152" />
|
||||
<path d="M 88 116 Q 80 152 70 174" />
|
||||
<path d="M 112 116 Q 120 152 130 174" />
|
||||
</g>
|
||||
<path
|
||||
d="M 134 108 C 162 116, 172 96, 162 78 C 154 64, 168 56, 178 60"
|
||||
fill="none"
|
||||
stroke="#FF8A4C"
|
||||
strokeWidth="16"
|
||||
strokeLinecap="round"
|
||||
/>
|
||||
<path
|
||||
d="M 48 102 C 48 56, 78 30, 100 30 C 122 30, 152 56, 152 102 C 152 116, 132 120, 100 120 C 68 120, 48 116, 48 102 Z"
|
||||
fill={body}
|
||||
/>
|
||||
<path
|
||||
d="M 80 84 Q 86 77 92 84"
|
||||
fill="none"
|
||||
stroke={eye}
|
||||
strokeWidth="3.5"
|
||||
strokeLinecap="round"
|
||||
/>
|
||||
<path
|
||||
d="M 108 84 Q 114 77 120 84"
|
||||
fill="none"
|
||||
stroke={eye}
|
||||
strokeWidth="3.5"
|
||||
strokeLinecap="round"
|
||||
/>
|
||||
</svg>
|
||||
);
|
||||
}
|
||||
|
|
@ -1,493 +0,0 @@
|
|||
"use client";
|
||||
|
||||
import { Handle, Position, type Node, type NodeProps } from "@xyflow/react";
|
||||
|
||||
import { KtxMascot } from "./mascot";
|
||||
|
||||
/** Fixed palette mirrored from the approved SVG diagrams so the exported PNG
|
||||
* is theme-independent (one image that reads on light and dark GitHub). */
|
||||
export const C = {
|
||||
ink: "#1b1b18",
|
||||
inkSoft: "#57534e",
|
||||
inkMuted: "#8c857f",
|
||||
cardBorder: "#e2dfd9",
|
||||
engineBg: "#15323a",
|
||||
engineBorder: "#23474f",
|
||||
cyan: "#55dced",
|
||||
stepNum: "#06262c",
|
||||
stepTitle: "#f3f1ec",
|
||||
stepDesc: "#9fb6bc",
|
||||
hubRow: "#eef4f5",
|
||||
chipBg: "#faf9f6",
|
||||
chipBorder: "#e7e5e4",
|
||||
teal: "#0e7490",
|
||||
emerald: "#059669",
|
||||
orange: "#f97316",
|
||||
amber: "#d97706",
|
||||
slate: "#334155",
|
||||
neutral: "#94a3b8",
|
||||
} as const;
|
||||
|
||||
const DISPLAY = "var(--font-display), system-ui, sans-serif";
|
||||
const BODY = "var(--font-inter), system-ui, sans-serif";
|
||||
const MONO = "var(--font-mono), ui-monospace, monospace";
|
||||
|
||||
const CARD_SHADOW = "0 3px 12px rgba(27, 49, 57, 0.10)";
|
||||
const ENGINE_SHADOW = "0 6px 22px rgba(2, 12, 15, 0.30)";
|
||||
|
||||
/** ktx logo mascot size, shared by the engine and hub headers. */
|
||||
const LOGO_SIZE = 56;
|
||||
|
||||
type HandleSpec = {
|
||||
side: "left" | "right";
|
||||
type: "source" | "target";
|
||||
id: string;
|
||||
top?: string;
|
||||
};
|
||||
|
||||
function Handles({ specs }: { specs?: HandleSpec[] }) {
|
||||
if (!specs) return null;
|
||||
return (
|
||||
<>
|
||||
{specs.map((h) => (
|
||||
<Handle
|
||||
key={`${h.type}-${h.id}`}
|
||||
id={h.id}
|
||||
type={h.type}
|
||||
position={h.side === "left" ? Position.Left : Position.Right}
|
||||
isConnectable={false}
|
||||
style={{
|
||||
opacity: 0,
|
||||
border: 0,
|
||||
background: "transparent",
|
||||
...(h.top ? { top: h.top } : {}),
|
||||
}}
|
||||
/>
|
||||
))}
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
/* ------------------------------- Card node ------------------------------- */
|
||||
|
||||
type CardRow =
|
||||
| { kind: "title"; text: string }
|
||||
| { kind: "mono"; text: string; color: string }
|
||||
| { kind: "desc"; text: string }
|
||||
| { kind: "muted"; text: string }
|
||||
| { kind: "chips"; items: string[] }
|
||||
| { kind: "badge"; text: string; bg: string; border: string; color: string };
|
||||
|
||||
type CardData = {
|
||||
width: number;
|
||||
height: number;
|
||||
accent: string;
|
||||
align?: "center";
|
||||
rows: CardRow[];
|
||||
handles?: HandleSpec[];
|
||||
};
|
||||
|
||||
function gapFor(kind: CardRow["kind"], prev?: CardRow["kind"]): number {
|
||||
if (!prev) return 0;
|
||||
if (kind === "desc" && prev === "desc") return 3;
|
||||
if (kind === "mono" && prev === "mono") return 2;
|
||||
if (kind === "title") return 6;
|
||||
return 10;
|
||||
}
|
||||
|
||||
function CardRowView({ row }: { row: CardRow }) {
|
||||
switch (row.kind) {
|
||||
case "title":
|
||||
return (
|
||||
<span
|
||||
style={{
|
||||
fontFamily: DISPLAY,
|
||||
fontWeight: 700,
|
||||
fontSize: 26,
|
||||
lineHeight: 1.15,
|
||||
color: C.ink,
|
||||
}}
|
||||
>
|
||||
{row.text}
|
||||
</span>
|
||||
);
|
||||
case "mono":
|
||||
return (
|
||||
<span
|
||||
style={{
|
||||
fontFamily: MONO,
|
||||
fontWeight: 700,
|
||||
fontSize: 18,
|
||||
lineHeight: 1.4,
|
||||
color: row.color,
|
||||
}}
|
||||
>
|
||||
{row.text}
|
||||
</span>
|
||||
);
|
||||
case "desc":
|
||||
return (
|
||||
<span
|
||||
style={{
|
||||
fontFamily: BODY,
|
||||
fontWeight: 500,
|
||||
fontSize: 17,
|
||||
lineHeight: 1.45,
|
||||
color: C.inkSoft,
|
||||
}}
|
||||
>
|
||||
{row.text}
|
||||
</span>
|
||||
);
|
||||
case "muted":
|
||||
return (
|
||||
<span
|
||||
style={{
|
||||
fontFamily: BODY,
|
||||
fontWeight: 500,
|
||||
fontSize: 14,
|
||||
lineHeight: 1.4,
|
||||
color: C.inkMuted,
|
||||
}}
|
||||
>
|
||||
{row.text}
|
||||
</span>
|
||||
);
|
||||
case "chips":
|
||||
return (
|
||||
<div style={{ display: "flex", gap: 8, flexWrap: "wrap" }}>
|
||||
{row.items.map((c) => (
|
||||
<span
|
||||
key={c}
|
||||
style={{
|
||||
fontFamily: BODY,
|
||||
fontWeight: 600,
|
||||
fontSize: 14,
|
||||
color: C.inkSoft,
|
||||
background: C.chipBg,
|
||||
border: `1px solid ${C.chipBorder}`,
|
||||
borderRadius: 6,
|
||||
padding: "4px 10px",
|
||||
}}
|
||||
>
|
||||
{c}
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
case "badge":
|
||||
return (
|
||||
<span
|
||||
style={{
|
||||
display: "inline-flex",
|
||||
alignItems: "center",
|
||||
borderRadius: 14,
|
||||
padding: "3px 12px",
|
||||
fontFamily: BODY,
|
||||
fontWeight: 700,
|
||||
fontSize: 14,
|
||||
background: row.bg,
|
||||
border: `1px solid ${row.border}`,
|
||||
color: row.color,
|
||||
}}
|
||||
>
|
||||
{row.text}
|
||||
</span>
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function CardNode({ data }: NodeProps<Node<CardData>>) {
|
||||
const center = data.align === "center";
|
||||
return (
|
||||
<div
|
||||
style={{
|
||||
width: data.width,
|
||||
height: data.height,
|
||||
position: "relative",
|
||||
background: "#ffffff",
|
||||
border: `1px solid ${C.cardBorder}`,
|
||||
borderRadius: 10,
|
||||
boxShadow: CARD_SHADOW,
|
||||
padding: "18px 20px",
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
alignItems: center ? "center" : "flex-start",
|
||||
justifyContent: center ? "center" : "flex-start",
|
||||
textAlign: center ? "center" : "left",
|
||||
overflow: "hidden",
|
||||
}}
|
||||
>
|
||||
<span
|
||||
style={{
|
||||
position: "absolute",
|
||||
top: 0,
|
||||
left: 2,
|
||||
right: 2,
|
||||
height: 4,
|
||||
borderRadius: 2,
|
||||
background: data.accent,
|
||||
}}
|
||||
/>
|
||||
<Handles specs={data.handles} />
|
||||
{data.rows.map((row, i) => (
|
||||
<div
|
||||
key={i}
|
||||
style={{ marginTop: gapFor(row.kind, data.rows[i - 1]?.kind) }}
|
||||
>
|
||||
<CardRowView row={row} />
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
/* ------------------------------ Engine node ------------------------------ */
|
||||
|
||||
type EngineStep = { n: number; title: string; desc: string };
|
||||
|
||||
type EngineData = {
|
||||
width: number;
|
||||
height: number;
|
||||
steps: EngineStep[];
|
||||
handles?: HandleSpec[];
|
||||
};
|
||||
|
||||
function EngineNode({ data }: NodeProps<Node<EngineData>>) {
|
||||
return (
|
||||
<div
|
||||
style={{
|
||||
width: data.width,
|
||||
height: data.height,
|
||||
position: "relative",
|
||||
background: C.engineBg,
|
||||
border: `1px solid ${C.engineBorder}`,
|
||||
borderRadius: 14,
|
||||
boxShadow: ENGINE_SHADOW,
|
||||
padding: "24px 24px",
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
overflow: "hidden",
|
||||
}}
|
||||
>
|
||||
<span
|
||||
style={{
|
||||
position: "absolute",
|
||||
top: 0,
|
||||
left: 2,
|
||||
right: 2,
|
||||
height: 4,
|
||||
borderRadius: 2,
|
||||
background: C.cyan,
|
||||
}}
|
||||
/>
|
||||
<Handles specs={data.handles} />
|
||||
<div style={{ display: "flex", alignItems: "center", gap: 14 }}>
|
||||
<KtxMascot variant="dark" size={LOGO_SIZE} />
|
||||
<span
|
||||
style={{
|
||||
fontFamily: DISPLAY,
|
||||
fontWeight: 700,
|
||||
fontSize: 30,
|
||||
color: C.stepTitle,
|
||||
}}
|
||||
>
|
||||
ktx
|
||||
</span>
|
||||
</div>
|
||||
<div
|
||||
style={{
|
||||
flex: 1,
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
justifyContent: "space-around",
|
||||
marginTop: 6,
|
||||
}}
|
||||
>
|
||||
{data.steps.map((s) => (
|
||||
<div
|
||||
key={s.n}
|
||||
style={{ display: "flex", alignItems: "center", gap: 18 }}
|
||||
>
|
||||
<span
|
||||
style={{
|
||||
flex: "none",
|
||||
width: 44,
|
||||
height: 44,
|
||||
borderRadius: "50%",
|
||||
background: C.cyan,
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
justifyContent: "center",
|
||||
fontFamily: DISPLAY,
|
||||
fontWeight: 800,
|
||||
fontSize: 22,
|
||||
color: C.stepNum,
|
||||
}}
|
||||
>
|
||||
{s.n}
|
||||
</span>
|
||||
<div style={{ display: "flex", flexDirection: "column", gap: 3 }}>
|
||||
<span
|
||||
style={{
|
||||
fontFamily: DISPLAY,
|
||||
fontWeight: 700,
|
||||
fontSize: 24,
|
||||
lineHeight: 1.1,
|
||||
color: C.stepTitle,
|
||||
}}
|
||||
>
|
||||
{s.title}
|
||||
</span>
|
||||
<span
|
||||
style={{
|
||||
fontFamily: BODY,
|
||||
fontWeight: 500,
|
||||
fontSize: 16,
|
||||
lineHeight: 1.3,
|
||||
color: C.stepDesc,
|
||||
}}
|
||||
>
|
||||
{s.desc}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
/* -------------------------------- Hub node ------------------------------- */
|
||||
|
||||
type HubData = {
|
||||
width: number;
|
||||
height: number;
|
||||
rows: string[];
|
||||
handles?: HandleSpec[];
|
||||
};
|
||||
|
||||
function HubNode({ data }: NodeProps<Node<HubData>>) {
|
||||
return (
|
||||
<div
|
||||
style={{
|
||||
width: data.width,
|
||||
height: data.height,
|
||||
position: "relative",
|
||||
background: C.engineBg,
|
||||
border: `1px solid ${C.engineBorder}`,
|
||||
borderRadius: 14,
|
||||
boxShadow: ENGINE_SHADOW,
|
||||
padding: "24px 24px",
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
overflow: "hidden",
|
||||
}}
|
||||
>
|
||||
<span
|
||||
style={{
|
||||
position: "absolute",
|
||||
top: 0,
|
||||
left: 2,
|
||||
right: 2,
|
||||
height: 4,
|
||||
borderRadius: 2,
|
||||
background: C.cyan,
|
||||
}}
|
||||
/>
|
||||
<Handles specs={data.handles} />
|
||||
<div style={{ display: "flex", alignItems: "center", gap: 14 }}>
|
||||
<KtxMascot variant="dark" size={LOGO_SIZE} />
|
||||
<span
|
||||
style={{
|
||||
fontFamily: DISPLAY,
|
||||
fontWeight: 700,
|
||||
fontSize: 30,
|
||||
color: C.stepTitle,
|
||||
}}
|
||||
>
|
||||
ktx
|
||||
</span>
|
||||
</div>
|
||||
<div
|
||||
style={{
|
||||
marginTop: 22,
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
gap: 18,
|
||||
}}
|
||||
>
|
||||
{data.rows.map((r) => (
|
||||
<div key={r} style={{ display: "flex", alignItems: "center", gap: 14 }}>
|
||||
<span
|
||||
style={{
|
||||
flex: "none",
|
||||
width: 10,
|
||||
height: 10,
|
||||
borderRadius: "50%",
|
||||
background: C.cyan,
|
||||
}}
|
||||
/>
|
||||
<span
|
||||
style={{
|
||||
fontFamily: BODY,
|
||||
fontWeight: 600,
|
||||
fontSize: 19,
|
||||
color: C.hubRow,
|
||||
}}
|
||||
>
|
||||
{r}
|
||||
</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
/* ------------------------------- Title node ------------------------------ */
|
||||
|
||||
type TitleData = { width: number; eyebrow: string; title: string };
|
||||
|
||||
function TitleNode({ data }: NodeProps<Node<TitleData>>) {
|
||||
return (
|
||||
<div
|
||||
style={{
|
||||
width: data.width,
|
||||
display: "flex",
|
||||
flexDirection: "column",
|
||||
gap: 6,
|
||||
}}
|
||||
>
|
||||
<span
|
||||
style={{
|
||||
fontFamily: BODY,
|
||||
fontSize: 19,
|
||||
fontWeight: 800,
|
||||
letterSpacing: 2,
|
||||
textTransform: "uppercase",
|
||||
color: C.teal,
|
||||
}}
|
||||
>
|
||||
{data.eyebrow}
|
||||
</span>
|
||||
<span
|
||||
style={{
|
||||
fontFamily: DISPLAY,
|
||||
fontSize: 24,
|
||||
fontWeight: 600,
|
||||
color: C.inkMuted,
|
||||
}}
|
||||
>
|
||||
{data.title}
|
||||
</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export const nodeTypes = {
|
||||
card: CardNode,
|
||||
engine: EngineNode,
|
||||
hub: HubNode,
|
||||
title: TitleNode,
|
||||
};
|
||||
|
|
@ -1,242 +0,0 @@
|
|||
"use client";
|
||||
|
||||
import "@xyflow/react/dist/style.css";
|
||||
|
||||
import { useCallback, useRef, useState } from "react";
|
||||
import {
|
||||
Background,
|
||||
BackgroundVariant,
|
||||
type Edge,
|
||||
getNodesBounds,
|
||||
type Node,
|
||||
ReactFlow,
|
||||
ReactFlowProvider,
|
||||
useEdgesState,
|
||||
useNodesState,
|
||||
useReactFlow,
|
||||
} from "@xyflow/react";
|
||||
import { toPng } from "html-to-image";
|
||||
|
||||
import {
|
||||
ingestionEdges,
|
||||
ingestionNodes,
|
||||
runtimeEdges,
|
||||
runtimeNodes,
|
||||
} from "./flows";
|
||||
import { nodeTypes } from "./nodes";
|
||||
|
||||
const EXPORT_PADDING = 48;
|
||||
const EXPORT_PIXEL_RATIO = 2;
|
||||
|
||||
function DiagramCanvasInner({
|
||||
initialNodes,
|
||||
initialEdges,
|
||||
fileName,
|
||||
height,
|
||||
dark,
|
||||
}: {
|
||||
initialNodes: Node[];
|
||||
initialEdges: Edge[];
|
||||
fileName: string;
|
||||
height: number;
|
||||
dark: boolean;
|
||||
}) {
|
||||
const wrapperRef = useRef<HTMLDivElement>(null);
|
||||
const [nodes, , onNodesChange] = useNodesState(initialNodes);
|
||||
const [edges, , onEdgesChange] = useEdgesState(initialEdges);
|
||||
const { getNodes } = useReactFlow();
|
||||
const [busy, setBusy] = useState(false);
|
||||
|
||||
const download = useCallback(async () => {
|
||||
const viewport = wrapperRef.current?.querySelector<HTMLElement>(
|
||||
".react-flow__viewport",
|
||||
);
|
||||
if (!viewport) return;
|
||||
setBusy(true);
|
||||
try {
|
||||
await document.fonts.ready;
|
||||
const bounds = getNodesBounds(getNodes());
|
||||
const outW = Math.ceil(bounds.width + EXPORT_PADDING * 2);
|
||||
const outH = Math.ceil(bounds.height + EXPORT_PADDING * 2);
|
||||
const tx = EXPORT_PADDING - bounds.x;
|
||||
const ty = EXPORT_PADDING - bounds.y;
|
||||
const dataUrl = await toPng(viewport, {
|
||||
width: outW,
|
||||
height: outH,
|
||||
pixelRatio: EXPORT_PIXEL_RATIO,
|
||||
// transparent background so one PNG works on light and dark GitHub
|
||||
style: {
|
||||
width: `${outW}px`,
|
||||
height: `${outH}px`,
|
||||
transform: `translate(${tx}px, ${ty}px) scale(1)`,
|
||||
},
|
||||
});
|
||||
const link = document.createElement("a");
|
||||
link.download = fileName;
|
||||
link.href = dataUrl;
|
||||
link.click();
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
}, [fileName, getNodes]);
|
||||
|
||||
return (
|
||||
<div>
|
||||
<div style={{ display: "flex", gap: 8, marginBottom: 10 }}>
|
||||
<button
|
||||
type="button"
|
||||
onClick={download}
|
||||
disabled={busy}
|
||||
style={btnStyle(busy)}
|
||||
>
|
||||
{busy ? "Exporting…" : "Download PNG"}
|
||||
</button>
|
||||
</div>
|
||||
<div
|
||||
ref={wrapperRef}
|
||||
style={{
|
||||
height,
|
||||
borderRadius: 12,
|
||||
border: "1px solid rgba(127,127,127,0.2)",
|
||||
background: dark ? "#0d1117" : "#ffffff",
|
||||
}}
|
||||
>
|
||||
<ReactFlow
|
||||
nodes={nodes}
|
||||
edges={edges}
|
||||
nodeTypes={nodeTypes}
|
||||
onNodesChange={onNodesChange}
|
||||
onEdgesChange={onEdgesChange}
|
||||
fitView
|
||||
fitViewOptions={{ padding: 0.08 }}
|
||||
nodesDraggable={false}
|
||||
nodesConnectable={false}
|
||||
nodesFocusable={false}
|
||||
edgesFocusable={false}
|
||||
elementsSelectable={false}
|
||||
panOnDrag={false}
|
||||
panOnScroll={false}
|
||||
zoomOnScroll={false}
|
||||
zoomOnPinch={false}
|
||||
zoomOnDoubleClick={false}
|
||||
preventScrolling={false}
|
||||
proOptions={{ hideAttribution: true }}
|
||||
>
|
||||
<Background
|
||||
variant={BackgroundVariant.Dots}
|
||||
gap={18}
|
||||
size={1}
|
||||
color={dark ? "#1f2a30" : "#e6e2db"}
|
||||
/>
|
||||
</ReactFlow>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function btnStyle(disabled: boolean): React.CSSProperties {
|
||||
return {
|
||||
fontFamily: "var(--font-inter), system-ui, sans-serif",
|
||||
fontSize: 13,
|
||||
fontWeight: 600,
|
||||
padding: "7px 14px",
|
||||
borderRadius: 8,
|
||||
border: "1px solid #0e7490",
|
||||
background: disabled ? "#9bbdc6" : "#0e7490",
|
||||
color: "#ffffff",
|
||||
cursor: disabled ? "default" : "pointer",
|
||||
};
|
||||
}
|
||||
|
||||
function DiagramCanvas(props: {
|
||||
initialNodes: Node[];
|
||||
initialEdges: Edge[];
|
||||
fileName: string;
|
||||
height: number;
|
||||
dark: boolean;
|
||||
}) {
|
||||
return (
|
||||
<ReactFlowProvider>
|
||||
<DiagramCanvasInner {...props} />
|
||||
</ReactFlowProvider>
|
||||
);
|
||||
}
|
||||
|
||||
export function DiagramStudio() {
|
||||
const [dark, setDark] = useState(false);
|
||||
return (
|
||||
<main
|
||||
style={{
|
||||
maxWidth: 1320,
|
||||
margin: "0 auto",
|
||||
padding: "32px 24px 80px",
|
||||
fontFamily: "var(--font-inter), system-ui, sans-serif",
|
||||
}}
|
||||
>
|
||||
<header style={{ marginBottom: 24 }}>
|
||||
<h1
|
||||
style={{
|
||||
fontFamily: "var(--font-display), system-ui, sans-serif",
|
||||
fontSize: 30,
|
||||
fontWeight: 700,
|
||||
color: "#1b1b18",
|
||||
margin: 0,
|
||||
}}
|
||||
>
|
||||
ktx diagram studio
|
||||
</h1>
|
||||
<p style={{ color: "#6b6560", marginTop: 6, fontSize: 15 }}>
|
||||
Static diagrams. Export is a transparent 2× PNG framed to the node
|
||||
bounds — the dark-background toggle is only for previewing.
|
||||
</p>
|
||||
<label
|
||||
style={{
|
||||
display: "inline-flex",
|
||||
alignItems: "center",
|
||||
gap: 8,
|
||||
marginTop: 12,
|
||||
fontSize: 14,
|
||||
color: "#57534e",
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={dark}
|
||||
onChange={(e) => setDark(e.target.checked)}
|
||||
/>
|
||||
Preview on dark background
|
||||
</label>
|
||||
</header>
|
||||
|
||||
<section style={{ marginBottom: 40 }}>
|
||||
<h2 style={sectionTitle}>1 · Ingestion — building the context layer</h2>
|
||||
<DiagramCanvas
|
||||
initialNodes={ingestionNodes}
|
||||
initialEdges={ingestionEdges}
|
||||
fileName="ingestion-flow.png"
|
||||
height={560}
|
||||
dark={dark}
|
||||
/>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2 style={sectionTitle}>2 · Serving — answering agents at runtime</h2>
|
||||
<DiagramCanvas
|
||||
initialNodes={runtimeNodes}
|
||||
initialEdges={runtimeEdges}
|
||||
fileName="mcp-runtime-flow.png"
|
||||
height={480}
|
||||
dark={dark}
|
||||
/>
|
||||
</section>
|
||||
</main>
|
||||
);
|
||||
}
|
||||
|
||||
const sectionTitle: React.CSSProperties = {
|
||||
fontFamily: "var(--font-display), system-ui, sans-serif",
|
||||
fontSize: 18,
|
||||
fontWeight: 600,
|
||||
color: "#1b1b18",
|
||||
marginBottom: 12,
|
||||
};
|
||||
|
|
@ -1,56 +1,40 @@
|
|||
"use client";
|
||||
|
||||
import Link from "next/link";
|
||||
|
||||
const brandFont = {
|
||||
fontFamily: "var(--font-display), var(--font-sans), sans-serif",
|
||||
} as const;
|
||||
|
||||
export function Logo({ href = "/", className }: { href?: string; className?: string }) {
|
||||
export function Logo() {
|
||||
return (
|
||||
<div className={className}>
|
||||
<div className="flex items-center gap-3.5 group">
|
||||
<Link href={href} aria-label="ktx documentation home" className="flex items-center no-underline">
|
||||
<span className="relative flex items-center justify-center transition-transform duration-300 ease-out group-hover:rotate-[-4deg]">
|
||||
<img
|
||||
src="/ktx/brand/ktx-mascot.svg"
|
||||
alt=""
|
||||
aria-hidden="true"
|
||||
className="h-20 w-20 object-contain block dark:hidden"
|
||||
/>
|
||||
<img
|
||||
src="/ktx/brand/ktx-mascot-dark.svg"
|
||||
alt=""
|
||||
aria-hidden="true"
|
||||
className="h-20 w-20 object-contain hidden dark:block"
|
||||
/>
|
||||
</span>
|
||||
</Link>
|
||||
<div className="flex flex-col items-start leading-none">
|
||||
<Link
|
||||
href={href}
|
||||
className="text-[42px] font-semibold text-fd-foreground tracking-tight no-underline"
|
||||
style={brandFont}
|
||||
>
|
||||
ktx
|
||||
</Link>
|
||||
<a
|
||||
href="https://www.kaelio.com"
|
||||
target="_blank"
|
||||
rel="noreferrer"
|
||||
className="mt-1 whitespace-nowrap text-[13px] font-medium text-fd-muted-foreground/80 tracking-tight no-underline transition-colors hover:text-fd-foreground"
|
||||
style={brandFont}
|
||||
>
|
||||
by Kaelio
|
||||
</a>
|
||||
</div>
|
||||
<div className="flex items-center gap-3.5 group">
|
||||
<div className="relative flex items-center justify-center transition-transform duration-300 ease-out group-hover:rotate-[-4deg]">
|
||||
<img
|
||||
src="/ktx/brand/ktx-mascot.svg"
|
||||
alt=""
|
||||
aria-hidden="true"
|
||||
className="h-20 w-20 object-contain block dark:hidden"
|
||||
/>
|
||||
<img
|
||||
src="/ktx/brand/ktx-mascot-dark.svg"
|
||||
alt=""
|
||||
aria-hidden="true"
|
||||
className="h-20 w-20 object-contain hidden dark:block"
|
||||
/>
|
||||
</div>
|
||||
<div className="flex flex-col items-start leading-none">
|
||||
<span
|
||||
className="text-[19px] font-medium text-fd-muted-foreground/80 tracking-tight border-l border-fd-border pl-3 ml-1"
|
||||
style={brandFont}
|
||||
className="text-[42px] font-semibold text-fd-foreground tracking-tight"
|
||||
style={{ fontFamily: "var(--font-display), var(--font-sans), sans-serif" }}
|
||||
>
|
||||
Docs
|
||||
ktx
|
||||
</span>
|
||||
<span
|
||||
className="mt-1 whitespace-nowrap text-[13px] font-medium text-fd-muted-foreground/80 tracking-tight"
|
||||
style={{ fontFamily: "var(--font-display), var(--font-sans), sans-serif" }}
|
||||
>
|
||||
by Kaelio
|
||||
</span>
|
||||
</div>
|
||||
<span
|
||||
className="text-[19px] font-medium text-fd-muted-foreground/80 tracking-tight border-l border-fd-border pl-3 ml-1"
|
||||
style={{ fontFamily: "var(--font-display), var(--font-sans), sans-serif" }}
|
||||
>
|
||||
Docs
|
||||
</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,576 +0,0 @@
|
|||
"use client";
|
||||
|
||||
import {
|
||||
type Edge,
|
||||
type EdgeProps,
|
||||
getSmoothStepPath,
|
||||
Handle,
|
||||
MarkerType,
|
||||
type Node,
|
||||
type NodeProps,
|
||||
Position,
|
||||
} from "@xyflow/react";
|
||||
|
||||
import { FlowCanvas } from "./flow-canvas";
|
||||
|
||||
type AgentNodeData = {
|
||||
title: string;
|
||||
items: string[];
|
||||
};
|
||||
|
||||
type HubNodeData = {
|
||||
title: string;
|
||||
badge: string;
|
||||
rows: string[];
|
||||
};
|
||||
|
||||
type TargetNodeData = {
|
||||
accent: string;
|
||||
title: string;
|
||||
body: string;
|
||||
rows: { text: string; color?: string; mono?: boolean }[];
|
||||
badge?: string;
|
||||
};
|
||||
|
||||
type AgentNode = Node<AgentNodeData, "agent">;
|
||||
type HubNode = Node<HubNodeData, "hub">;
|
||||
type TargetNode = Node<TargetNodeData, "target">;
|
||||
type FlowNode = AgentNode | HubNode | TargetNode;
|
||||
|
||||
const AGENT_W = 252;
|
||||
const AGENT_H = 96;
|
||||
const HUB_W = 306;
|
||||
const HUB_H = 190;
|
||||
const TARGET_W = 268;
|
||||
const TARGET_H = 148;
|
||||
|
||||
const CENTER_X = 470;
|
||||
const ROW_AGENT_Y = 0;
|
||||
const ROW_HUB_Y = 196;
|
||||
const ROW_TARGET_Y = 488;
|
||||
|
||||
const AGENT_X = CENTER_X - AGENT_W / 2;
|
||||
const HUB_X = CENTER_X - HUB_W / 2;
|
||||
|
||||
const TARGET_GAP_X = 38;
|
||||
const TARGETS_TOTAL = TARGET_W * 2 + TARGET_GAP_X;
|
||||
const TARGETS_START_X = CENTER_X - TARGETS_TOTAL / 2;
|
||||
const CONTEXT_X = TARGETS_START_X;
|
||||
const WAREHOUSE_X = TARGETS_START_X + TARGET_W + TARGET_GAP_X;
|
||||
|
||||
const EDGE_STROKE = "#94a3b8";
|
||||
const CYCLE_STROKE = "#0e7490";
|
||||
const EMERALD = "#059669";
|
||||
const TEAL = "#0e7490";
|
||||
|
||||
const nodes: FlowNode[] = [
|
||||
{
|
||||
id: "agent",
|
||||
type: "agent",
|
||||
position: { x: AGENT_X, y: ROW_AGENT_Y },
|
||||
data: {
|
||||
title: "Your agent",
|
||||
items: ["Claude Code", "Cursor", "Codex"],
|
||||
},
|
||||
draggable: false,
|
||||
selectable: false,
|
||||
},
|
||||
{
|
||||
id: "hub",
|
||||
type: "hub",
|
||||
position: { x: HUB_X, y: ROW_HUB_Y },
|
||||
data: {
|
||||
title: "ktx",
|
||||
badge: "MCP + CLI",
|
||||
rows: [
|
||||
"Search wiki + semantic layer",
|
||||
"Return approved metrics",
|
||||
"Compile metrics → SQL",
|
||||
],
|
||||
},
|
||||
draggable: false,
|
||||
selectable: false,
|
||||
},
|
||||
{
|
||||
id: "context",
|
||||
type: "target",
|
||||
position: { x: CONTEXT_X, y: ROW_TARGET_Y },
|
||||
data: {
|
||||
accent: TEAL,
|
||||
title: "Context layer",
|
||||
body: "Approved definitions agents search before they answer.",
|
||||
rows: [
|
||||
{ text: "wiki/*.md", color: EMERALD, mono: true },
|
||||
{ text: "semantic-layer/*.yaml", color: TEAL, mono: true },
|
||||
],
|
||||
},
|
||||
draggable: false,
|
||||
selectable: false,
|
||||
},
|
||||
{
|
||||
id: "warehouse",
|
||||
type: "target",
|
||||
position: { x: WAREHOUSE_X, y: ROW_TARGET_Y },
|
||||
data: {
|
||||
accent: "#334155",
|
||||
title: "Database",
|
||||
badge: "read-only",
|
||||
body: "Runs the compiled SQL. ktx never writes to it.",
|
||||
rows: [],
|
||||
},
|
||||
draggable: false,
|
||||
selectable: false,
|
||||
},
|
||||
];
|
||||
|
||||
const labelBg = {
|
||||
labelBgPadding: [6, 3] as [number, number],
|
||||
labelBgBorderRadius: 4,
|
||||
labelStyle: {
|
||||
fontSize: 13,
|
||||
fontWeight: 600,
|
||||
fill: "var(--color-fd-muted-foreground)",
|
||||
},
|
||||
labelBgStyle: {
|
||||
fill: "var(--color-fd-background)",
|
||||
stroke: "var(--color-fd-border)",
|
||||
strokeWidth: 1,
|
||||
},
|
||||
};
|
||||
|
||||
const requestMarker = {
|
||||
type: MarkerType.ArrowClosed,
|
||||
color: EDGE_STROKE,
|
||||
width: 16,
|
||||
height: 16,
|
||||
};
|
||||
|
||||
const flowEdges: Edge[] = [
|
||||
{
|
||||
id: "e-ask",
|
||||
source: "agent",
|
||||
sourceHandle: "ask",
|
||||
target: "hub",
|
||||
targetHandle: "ask",
|
||||
type: "straight",
|
||||
label: "ask",
|
||||
...labelBg,
|
||||
style: { stroke: EDGE_STROKE, strokeWidth: 1.5 },
|
||||
markerEnd: requestMarker,
|
||||
},
|
||||
{
|
||||
id: "e-answer",
|
||||
source: "hub",
|
||||
sourceHandle: "answer",
|
||||
target: "agent",
|
||||
targetHandle: "answer",
|
||||
type: "straight",
|
||||
label: "answer",
|
||||
...labelBg,
|
||||
style: { stroke: EDGE_STROKE, strokeWidth: 1.5 },
|
||||
markerEnd: requestMarker,
|
||||
},
|
||||
{
|
||||
id: "e-search",
|
||||
source: "hub",
|
||||
sourceHandle: "to-context",
|
||||
target: "context",
|
||||
targetHandle: "in",
|
||||
type: "smoothstep",
|
||||
label: "search + read",
|
||||
...labelBg,
|
||||
style: { stroke: CYCLE_STROKE, strokeWidth: 1.5 },
|
||||
markerStart: { type: MarkerType.ArrowClosed, color: CYCLE_STROKE, width: 14, height: 14 },
|
||||
markerEnd: { type: MarkerType.ArrowClosed, color: CYCLE_STROKE, width: 14, height: 14 },
|
||||
},
|
||||
{
|
||||
id: "e-readonly",
|
||||
source: "hub",
|
||||
sourceHandle: "to-warehouse",
|
||||
target: "warehouse",
|
||||
targetHandle: "in",
|
||||
type: "smoothstep",
|
||||
label: "read-only",
|
||||
...labelBg,
|
||||
style: { stroke: CYCLE_STROKE, strokeWidth: 1.5 },
|
||||
markerStart: { type: MarkerType.ArrowClosed, color: CYCLE_STROKE, width: 14, height: 14 },
|
||||
markerEnd: { type: MarkerType.ArrowClosed, color: CYCLE_STROKE, width: 14, height: 14 },
|
||||
},
|
||||
];
|
||||
|
||||
function AgentNodeView({ data }: NodeProps<AgentNode>) {
|
||||
return (
|
||||
<div
|
||||
style={{ width: AGENT_W, height: AGENT_H }}
|
||||
className="flex flex-col justify-center rounded-md border border-fd-border bg-fd-card px-3.5 py-2.5 shadow-sm"
|
||||
>
|
||||
<Handle
|
||||
id="ask"
|
||||
type="source"
|
||||
position={Position.Bottom}
|
||||
className="!opacity-0"
|
||||
style={{ left: "35%" }}
|
||||
/>
|
||||
<Handle
|
||||
id="answer"
|
||||
type="target"
|
||||
position={Position.Bottom}
|
||||
className="!opacity-0"
|
||||
style={{ left: "65%" }}
|
||||
/>
|
||||
<div className="flex items-center gap-2.5">
|
||||
<span className="flex h-8 w-8 flex-none items-center justify-center rounded-full bg-fd-primary/15 text-fd-primary">
|
||||
<svg
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
width="18"
|
||||
height="18"
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
strokeWidth="1.75"
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
aria-hidden="true"
|
||||
>
|
||||
<rect x="3" y="6" width="18" height="12" rx="3" />
|
||||
<circle cx="9" cy="12" r="1.25" fill="currentColor" stroke="none" />
|
||||
<circle cx="15" cy="12" r="1.25" fill="currentColor" stroke="none" />
|
||||
<path d="M12 3v3" />
|
||||
</svg>
|
||||
</span>
|
||||
<p className="text-[17px] font-semibold leading-6 text-fd-foreground">
|
||||
{data.title}
|
||||
</p>
|
||||
</div>
|
||||
<div className="mt-2 flex flex-wrap gap-1.5">
|
||||
{data.items.map((item) => (
|
||||
<span
|
||||
key={item}
|
||||
className="rounded border border-fd-border bg-fd-background px-1.5 py-0.5 text-[12px] leading-5 text-fd-muted-foreground"
|
||||
>
|
||||
{item}
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function HubNodeView({ data }: NodeProps<HubNode>) {
|
||||
return (
|
||||
<div
|
||||
style={{ width: HUB_W, height: HUB_H }}
|
||||
className="relative flex flex-col rounded-md border border-cyan-200/20 bg-[#0f1f23] px-4 py-3.5 text-white shadow-sm dark:bg-[#0b181b]"
|
||||
>
|
||||
<Handle
|
||||
id="ask"
|
||||
type="target"
|
||||
position={Position.Top}
|
||||
className="!opacity-0"
|
||||
style={{ left: "37.5%" }}
|
||||
/>
|
||||
<Handle
|
||||
id="answer"
|
||||
type="source"
|
||||
position={Position.Top}
|
||||
className="!opacity-0"
|
||||
style={{ left: "62.5%" }}
|
||||
/>
|
||||
<Handle
|
||||
id="to-context"
|
||||
type="source"
|
||||
position={Position.Bottom}
|
||||
className="!opacity-0"
|
||||
style={{ left: "44%" }}
|
||||
/>
|
||||
<Handle
|
||||
id="to-warehouse"
|
||||
type="source"
|
||||
position={Position.Bottom}
|
||||
className="!opacity-0"
|
||||
style={{ left: "56%" }}
|
||||
/>
|
||||
<div className="flex items-center gap-2.5">
|
||||
<span className="flex h-7 w-7 flex-none items-center justify-center rounded-md bg-cyan-300/95 font-mono text-sm font-bold text-[#0b1c20]">
|
||||
k
|
||||
</span>
|
||||
<span className="text-[19px] font-bold leading-6 text-white">
|
||||
{data.title}
|
||||
</span>
|
||||
<span className="ml-1 rounded border border-cyan-200/30 bg-white/5 px-1.5 py-0.5 font-mono text-[11px] leading-5 text-cyan-100/85">
|
||||
{data.badge}
|
||||
</span>
|
||||
</div>
|
||||
<div className="mt-3 flex flex-1 flex-col justify-center gap-2">
|
||||
{data.rows.map((row) => (
|
||||
<div key={row} className="flex items-center gap-2.5">
|
||||
<span className="h-1.5 w-1.5 flex-none rounded-full bg-cyan-300/95" />
|
||||
<span className="text-[14px] font-medium leading-5 text-cyan-50/90">
|
||||
{row}
|
||||
</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function TargetNodeView({ data }: NodeProps<TargetNode>) {
|
||||
return (
|
||||
<div
|
||||
style={{
|
||||
width: TARGET_W,
|
||||
height: TARGET_H,
|
||||
borderTop: `3px solid ${data.accent}`,
|
||||
}}
|
||||
className="overflow-hidden rounded-md border border-fd-border bg-fd-card px-3.5 py-3 shadow-sm"
|
||||
>
|
||||
<Handle id="in" type="target" position={Position.Top} className="!opacity-0" />
|
||||
<div className="flex items-center gap-2">
|
||||
<p className="text-[17px] font-semibold leading-6 text-fd-foreground">
|
||||
{data.title}
|
||||
</p>
|
||||
{data.badge ? (
|
||||
<span
|
||||
className="rounded-full px-1.5 py-0.5 text-[11px] font-semibold leading-5"
|
||||
style={{
|
||||
color: data.accent,
|
||||
background: "color-mix(in oklch, var(--color-fd-card) 86%, #64748b)",
|
||||
}}
|
||||
>
|
||||
{data.badge}
|
||||
</span>
|
||||
) : null}
|
||||
</div>
|
||||
{data.rows.length > 0 ? (
|
||||
<div className="mt-1 flex flex-col gap-0.5">
|
||||
{data.rows.map((row) => (
|
||||
<span
|
||||
key={row.text}
|
||||
className={
|
||||
row.mono
|
||||
? "font-mono text-[13px] font-semibold tracking-tight"
|
||||
: "text-[12px] leading-4 text-fd-muted-foreground"
|
||||
}
|
||||
style={row.color ? { color: row.color } : undefined}
|
||||
>
|
||||
{row.text}
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
) : null}
|
||||
<p className="mt-1.5 line-clamp-2 text-[13px] leading-[18px] text-fd-muted-foreground">
|
||||
{data.body}
|
||||
</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
/* ------------------------------- Particles ------------------------------- */
|
||||
|
||||
const PARTICLE_SPEED_PX_PER_SEC = 150;
|
||||
const PARTICLE_MIN_DURATION_SEC = 5;
|
||||
|
||||
type Leg = {
|
||||
sx: number;
|
||||
sy: number;
|
||||
sPos: Position;
|
||||
tx: number;
|
||||
ty: number;
|
||||
tPos: Position;
|
||||
};
|
||||
|
||||
const AGENT_ASK_X = AGENT_X + AGENT_W * 0.35;
|
||||
const AGENT_ANSWER_X = AGENT_X + AGENT_W * 0.65;
|
||||
const AGENT_BOTTOM_Y = ROW_AGENT_Y + AGENT_H;
|
||||
const HUB_ASK_X = HUB_X + HUB_W * 0.375;
|
||||
const HUB_ANSWER_X = HUB_X + HUB_W * 0.625;
|
||||
const HUB_TO_CONTEXT_X = HUB_X + HUB_W * 0.44;
|
||||
const HUB_TO_WAREHOUSE_X = HUB_X + HUB_W * 0.56;
|
||||
const HUB_BOTTOM_Y = ROW_HUB_Y + HUB_H;
|
||||
const CONTEXT_TOP_X = CONTEXT_X + TARGET_W / 2;
|
||||
const WAREHOUSE_TOP_X = WAREHOUSE_X + TARGET_W / 2;
|
||||
|
||||
function buildCyclePath(spokeX: number, targetX: number): {
|
||||
d: string;
|
||||
length: number;
|
||||
} {
|
||||
const legs: Leg[] = [
|
||||
// agent → hub (ask, down)
|
||||
{ sx: AGENT_ASK_X, sy: AGENT_BOTTOM_Y, sPos: Position.Bottom, tx: HUB_ASK_X, ty: ROW_HUB_Y, tPos: Position.Top },
|
||||
// through the hub to its spoke handle (down, drawn behind the hub)
|
||||
{ sx: HUB_ASK_X, sy: ROW_HUB_Y, sPos: Position.Bottom, tx: spokeX, ty: HUB_BOTTOM_Y, tPos: Position.Top },
|
||||
// hub → target (down)
|
||||
{ sx: spokeX, sy: HUB_BOTTOM_Y, sPos: Position.Bottom, tx: targetX, ty: ROW_TARGET_Y, tPos: Position.Top },
|
||||
// target → hub (up)
|
||||
{ sx: targetX, sy: ROW_TARGET_Y, sPos: Position.Top, tx: spokeX, ty: HUB_BOTTOM_Y, tPos: Position.Bottom },
|
||||
// through the hub to its answer handle (up, drawn behind the hub)
|
||||
{ sx: spokeX, sy: HUB_BOTTOM_Y, sPos: Position.Top, tx: HUB_ANSWER_X, ty: ROW_HUB_Y, tPos: Position.Bottom },
|
||||
// hub → agent (answer, up)
|
||||
{ sx: HUB_ANSWER_X, sy: ROW_HUB_Y, sPos: Position.Top, tx: AGENT_ANSWER_X, ty: AGENT_BOTTOM_Y, tPos: Position.Bottom },
|
||||
];
|
||||
|
||||
const segments = legs.map((leg) => {
|
||||
const [segment] = getSmoothStepPath({
|
||||
sourceX: leg.sx,
|
||||
sourceY: leg.sy,
|
||||
sourcePosition: leg.sPos,
|
||||
targetX: leg.tx,
|
||||
targetY: leg.ty,
|
||||
targetPosition: leg.tPos,
|
||||
});
|
||||
return segment;
|
||||
});
|
||||
|
||||
let d = segments[0];
|
||||
for (let i = 1; i < segments.length; i += 1) {
|
||||
d += ` ${segments[i].replace(/^M/, "L")}`;
|
||||
}
|
||||
|
||||
const length = legs.reduce(
|
||||
(sum, leg) => sum + Math.abs(leg.tx - leg.sx) + Math.abs(leg.ty - leg.sy),
|
||||
0,
|
||||
);
|
||||
|
||||
return { d, length };
|
||||
}
|
||||
|
||||
type ParticleEdgeData = {
|
||||
d: string;
|
||||
duration: number;
|
||||
beginOffset: number;
|
||||
color: string;
|
||||
};
|
||||
|
||||
type ParticleEdge = Edge<ParticleEdgeData, "particle">;
|
||||
|
||||
function ParticleEdgeView({ id, data }: EdgeProps<ParticleEdge>) {
|
||||
if (!data) return null;
|
||||
const pathId = `runtime-particle-path-${id}`;
|
||||
return (
|
||||
<>
|
||||
<path id={pathId} d={data.d} fill="none" stroke="none" pointerEvents="none" />
|
||||
<g className="runtime-particle" style={{ color: data.color }}>
|
||||
<circle r={7.5} fill="currentColor" opacity={0.16} />
|
||||
<circle r={3.75} fill="currentColor" opacity={0.32} />
|
||||
<circle r={2.1} fill="currentColor" />
|
||||
<animateMotion
|
||||
dur={`${data.duration.toFixed(2)}s`}
|
||||
begin={`-${data.beginOffset.toFixed(2)}s`}
|
||||
repeatCount="indefinite"
|
||||
>
|
||||
<mpath href={`#${pathId}`} />
|
||||
</animateMotion>
|
||||
</g>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
function makeCycleEdge(
|
||||
id: string,
|
||||
source: string,
|
||||
spokeX: number,
|
||||
targetX: number,
|
||||
beginFraction: number,
|
||||
): ParticleEdge {
|
||||
const { d, length } = buildCyclePath(spokeX, targetX);
|
||||
const duration = Math.max(
|
||||
PARTICLE_MIN_DURATION_SEC,
|
||||
length / PARTICLE_SPEED_PX_PER_SEC,
|
||||
);
|
||||
return {
|
||||
id,
|
||||
source,
|
||||
target: source,
|
||||
type: "particle",
|
||||
data: { d, duration, beginOffset: duration * beginFraction, color: CYCLE_STROKE },
|
||||
};
|
||||
}
|
||||
|
||||
const particleEdges: ParticleEdge[] = [
|
||||
makeCycleEdge("p-context", "context", HUB_TO_CONTEXT_X, CONTEXT_TOP_X, 0),
|
||||
makeCycleEdge("p-warehouse", "warehouse", HUB_TO_WAREHOUSE_X, WAREHOUSE_TOP_X, 0.5),
|
||||
];
|
||||
|
||||
const nodeTypes = {
|
||||
agent: AgentNodeView,
|
||||
hub: HubNodeView,
|
||||
target: TargetNodeView,
|
||||
};
|
||||
|
||||
const edgeTypes = {
|
||||
particle: ParticleEdgeView,
|
||||
};
|
||||
|
||||
const edges = [...flowEdges, ...particleEdges];
|
||||
|
||||
export function ProductRuntime() {
|
||||
return (
|
||||
<section
|
||||
className="not-prose my-12 w-full max-w-full min-w-0 space-y-5"
|
||||
aria-labelledby="runtime-title"
|
||||
>
|
||||
<div className="max-w-3xl">
|
||||
<h2
|
||||
id="runtime-title"
|
||||
className="text-xl font-semibold tracking-normal text-fd-foreground sm:text-2xl"
|
||||
style={{ fontFamily: "var(--font-display)" }}
|
||||
>
|
||||
How serving works
|
||||
</h2>
|
||||
<p className="mt-3 text-sm leading-6 text-fd-muted-foreground">
|
||||
At runtime, agents reach ktx through MCP. ktx searches the context
|
||||
layer, returns approved metrics, and compiles them into read-only SQL
|
||||
the warehouse runs.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<article
|
||||
className="max-w-full min-w-0 overflow-hidden rounded-lg border border-fd-border bg-fd-card shadow-sm"
|
||||
aria-label="ktx serving flow from an agent request to a governed answer"
|
||||
>
|
||||
<div className="border-b border-fd-border bg-fd-muted/35 px-5 py-4">
|
||||
<p className="text-xs font-semibold uppercase tracking-wide text-fd-primary">
|
||||
Serving flow
|
||||
</p>
|
||||
<h3
|
||||
className="mt-1 text-base font-semibold tracking-normal text-fd-foreground sm:text-lg"
|
||||
style={{ fontFamily: "var(--font-display)" }}
|
||||
>
|
||||
From an agent request to a governed answer
|
||||
</h3>
|
||||
<p className="mt-2 max-w-3xl text-xs leading-5 text-fd-muted-foreground">
|
||||
The agent asks in plain language. ktx is the only thing that touches
|
||||
the context layer and the warehouse, and every database connection
|
||||
is read-only.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<FlowCanvas
|
||||
nodes={nodes}
|
||||
edges={edges}
|
||||
nodeTypes={nodeTypes}
|
||||
edgeTypes={edgeTypes}
|
||||
canvasStyle={{
|
||||
height: "min(620px, 98vw)",
|
||||
minHeight: 430,
|
||||
}}
|
||||
className="runtime-canvas"
|
||||
fitViewOptions={{ padding: 0.06 }}
|
||||
ariaLabel="ktx serving flow diagram"
|
||||
/>
|
||||
</article>
|
||||
<style>{`
|
||||
.runtime-canvas .runtime-particle {
|
||||
pointer-events: none;
|
||||
filter: drop-shadow(0 0 6px currentColor);
|
||||
}
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.runtime-canvas .runtime-particle {
|
||||
display: none;
|
||||
}
|
||||
}
|
||||
`}</style>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
|
@ -1,86 +0,0 @@
|
|||
---
|
||||
title: "ktx completion"
|
||||
description: "Print a shell completion script for tab completion."
|
||||
---
|
||||
|
||||
Print a shell completion script for **ktx**. Once installed, pressing <kbd>Tab</kbd>
|
||||
completes commands, subcommands, and flags, and - inside a **ktx** project - the
|
||||
names of things that already exist: semantic-layer source names for
|
||||
`ktx sl read` and `ktx sl validate`, wiki page keys for `ktx wiki read`, and
|
||||
configured connection ids for `ktx connection test`, `ktx ingest`, and
|
||||
`ktx sql`. This saves you from remembering exact source, page, or connection
|
||||
names.
|
||||
|
||||
## Command signature
|
||||
|
||||
```bash
|
||||
ktx completion <shell>
|
||||
```
|
||||
|
||||
`<shell>` must be `zsh` or `bash`. The command writes the script to stdout; it
|
||||
does not modify any files. Enable completion by evaluating the script in your
|
||||
shell startup file.
|
||||
|
||||
## Installation
|
||||
|
||||
Add the matching line to your shell startup file, then restart your shell (or
|
||||
`source` the file). `ktx` must be on your `PATH`.
|
||||
|
||||
```bash
|
||||
# zsh — add to ~/.zshrc
|
||||
eval "$(ktx completion zsh)"
|
||||
```
|
||||
|
||||
```bash
|
||||
# bash — add to ~/.bashrc
|
||||
eval "$(ktx completion bash)"
|
||||
```
|
||||
|
||||
To try it for the current session only, run the same `eval` line directly in
|
||||
your terminal.
|
||||
|
||||
## What gets completed
|
||||
|
||||
| Position | Completions |
|
||||
|----------|-------------|
|
||||
| `ktx <Tab>` | Top-level commands (`setup`, `sl`, `wiki`, `ingest`, …) |
|
||||
| `ktx sl <Tab>` | The `read` / `validate` / `query` subcommands |
|
||||
| `ktx sl read <Tab>` | Existing semantic-layer source names |
|
||||
| `ktx sl validate <Tab>` | Existing semantic-layer source names |
|
||||
| `ktx wiki <Tab>` | The `read` subcommand |
|
||||
| `ktx wiki read <Tab>` | Existing wiki page keys |
|
||||
| `ktx connection test <Tab>` | Configured connection ids |
|
||||
| `ktx ingest <Tab>` | Configured connection ids |
|
||||
| `ktx sql --connection <Tab>` | Configured connection ids |
|
||||
| `ktx completion <Tab>` | `zsh` or `bash` |
|
||||
| `ktx <command> --<Tab>` | The command's flags and inherited global flags |
|
||||
| `ktx sl --output <Tab>` | An option's allowed values (here `pretty`, `plain`, `json`) |
|
||||
| `ktx sl --connection-id <Tab>` | Configured connection ids |
|
||||
|
||||
Source names, wiki page keys, and connection ids are read from the **ktx**
|
||||
project resolved from your current directory (or `--project-dir` /
|
||||
`KTX_PROJECT_DIR`). Outside a **ktx** project, completion still suggests
|
||||
commands and flags but no project entities. Bare `ktx sl <Tab>` and
|
||||
`ktx wiki <Tab>` complete subcommands instead of entity names because their
|
||||
positional arguments are free-text search queries.
|
||||
|
||||
## Examples
|
||||
|
||||
```bash
|
||||
# Print the zsh completion script
|
||||
ktx completion zsh
|
||||
|
||||
# Print the bash completion script
|
||||
ktx completion bash
|
||||
|
||||
# Install for zsh
|
||||
echo 'eval "$(ktx completion zsh)"' >> ~/.zshrc
|
||||
```
|
||||
|
||||
## Common errors
|
||||
|
||||
| Error | Cause | Recovery |
|
||||
|-------|-------|----------|
|
||||
| `error: command-argument value '<name>' is invalid for argument 'shell'. Allowed choices are zsh, bash.` | A shell other than `zsh` or `bash` was requested | Re-run with `ktx completion zsh` or `ktx completion bash` |
|
||||
| Tab completion does nothing | The script was not evaluated, or `ktx` is not on `PATH` | Confirm the `eval` line is in your startup file, restart the shell, and verify `ktx --version` runs |
|
||||
| Source, page, or connection names are missing | The current directory is not inside a **ktx** project | Run from the project directory, or pass `--project-dir`, or set `KTX_PROJECT_DIR` |
|
||||
|
|
@ -104,6 +104,6 @@ configured connection and exit non-zero if any probe fails.
|
|||
| Error | Cause | Recovery |
|
||||
|-------|-------|----------|
|
||||
| No connections configured | The project has no entries under `connections` | Run `ktx setup` and add a database or context-source connection |
|
||||
| Connection test fails | Credentials, network access, database, warehouse, or schema is invalid | Use the setup recovery menu to retry or re-enter details; if it still fails, verify the same URL with the database's native client |
|
||||
| Mapping validation fails during setup | BI database mappings do not point at valid warehouse connections | Use the setup recovery menu to retry validation or re-enter mapping selections; rerun `ktx setup` if you already exited |
|
||||
| Connection test fails | Credentials, network access, database, warehouse, or schema is invalid | Verify the same URL with the database's native client, then rerun `ktx setup` and reconfigure the connection |
|
||||
| Mapping validation fails during setup | BI database mappings do not point at valid warehouse connections | Rerun `ktx setup` and update the context-source mapping selections |
|
||||
| Notion page picker cannot run | The terminal is non-interactive or Notion discovery failed | Rerun interactive `ktx setup`, or use non-interactive setup flags with explicit root page ids |
|
||||
|
|
|
|||
|
|
@ -5,11 +5,9 @@ description: "Build or refresh ktx context, or capture text into ktx memory."
|
|||
|
||||
`ktx ingest` builds or refreshes **ktx** context from configured connections, and
|
||||
can also capture free-form text into **ktx** memory. Database connections build
|
||||
enriched context — schema plus AI-generated descriptions, embeddings, and
|
||||
relationship evidence — and require a configured model and embeddings.
|
||||
Context-source connections ingest metadata from tools such as dbt, Looker,
|
||||
Metabase, MetricFlow, LookML, and Notion. Pass `--text` or `--file` to capture
|
||||
inline text or text files into memory instead.
|
||||
schema context. Context-source connections ingest metadata from tools such as
|
||||
dbt, Looker, Metabase, MetricFlow, LookML, and Notion. Pass `--text` or
|
||||
`--file` to capture inline text or text files into memory instead.
|
||||
|
||||
## Command signature
|
||||
|
||||
|
|
@ -31,6 +29,8 @@ connection is selected.
|
|||
| Flag | Description | Default |
|
||||
|------|-------------|---------|
|
||||
| `--all` | Ingest all configured connections (same as bare invocation) | `false` |
|
||||
| `--fast` | Use deterministic fast database ingest | Stored connection default, or `fast` |
|
||||
| `--deep` | Use deep database ingest with AI-generated descriptions, embeddings, and relationship evidence | Stored connection default, or `fast` |
|
||||
| `--query-history` | Include database query-history usage patterns | Stored connection default |
|
||||
| `--no-query-history` | Skip database query-history usage patterns for this run | Stored connection default |
|
||||
| `--query-history-window-days <days>` | BigQuery/Snowflake query-history lookback window for this run | Stored connection default |
|
||||
|
|
@ -44,12 +44,12 @@ connection is selected.
|
|||
| `--yes` | Install required managed runtime features without prompting | `false` |
|
||||
| `--no-input` | Disable interactive terminal input | - |
|
||||
|
||||
Database ingest always builds enriched context and requires a configured model
|
||||
and embeddings (run `ktx setup`); connections without that configuration fail
|
||||
before any work starts. Query-history flags apply only to database connections
|
||||
`--fast` and `--deep` are mutually exclusive. Depth flags apply only to
|
||||
database connections. Query-history flags apply only to database connections
|
||||
that support query history. The window flag applies to BigQuery and Snowflake;
|
||||
Postgres reads the current `pg_stat_statements` aggregate data instead of a
|
||||
time-windowed history table. Query-history ingest runs after the schema scan.
|
||||
time-windowed history table. Query-history ingest runs after fast ingest and
|
||||
requires deep ingest readiness.
|
||||
|
||||
When more than one connection is selected, database ingest runs first, then
|
||||
context-source ingest and memory updates run for context-source connections.
|
||||
|
|
@ -72,8 +72,14 @@ ktx ingest
|
|||
# Build one database or context-source connection
|
||||
ktx ingest warehouse
|
||||
|
||||
# Force deterministic fast database ingest
|
||||
ktx ingest warehouse --fast
|
||||
|
||||
# Force deep database ingest with AI enrichment
|
||||
ktx ingest warehouse --deep
|
||||
|
||||
# Include query-history usage patterns
|
||||
ktx ingest warehouse --query-history
|
||||
ktx ingest warehouse --deep --query-history
|
||||
# Set the lookback window for BigQuery or Snowflake query history
|
||||
ktx ingest warehouse --query-history-window-days 30
|
||||
|
||||
|
|
@ -143,51 +149,13 @@ verbosity:
|
|||
KTX_INGEST_TRACE_LEVEL=trace ktx ingest metabase
|
||||
```
|
||||
|
||||
### Profiling a slow ingest
|
||||
|
||||
Each timed phase and work unit records a `durationMs` in the trace, and each
|
||||
agent loop records its step count and token usage. To see where wall-clock time
|
||||
went, enable profiling and **ktx** prints a rolled-up breakdown to stderr at the
|
||||
end of the run. There are two ways to turn it on, and two output formats.
|
||||
|
||||
Turn it on per run with the `KTX_PROFILE_INGEST` environment variable, or
|
||||
persistently with `ingest.profile` in `ktx.yaml` (useful for CI or while
|
||||
iterating on a slow source):
|
||||
|
||||
```bash
|
||||
KTX_PROFILE_INGEST=1 ktx ingest metabase # human-readable table
|
||||
KTX_PROFILE_INGEST=json ktx ingest metabase # raw JSON for coding agents
|
||||
```
|
||||
|
||||
```yaml
|
||||
ingest:
|
||||
profile: true # human table; use "json" for the machine-readable form
|
||||
```
|
||||
|
||||
Both formats report total wall time, time per phase, and the slowest work units,
|
||||
splitting each work unit's agent-loop time into model time versus tool-execution
|
||||
time. The `json` form emits the full structured profile (raw milliseconds and
|
||||
token counts, stable keys) plus a `summary.headline` one-line diagnosis, so a
|
||||
coding agent can parse it directly instead of scraping the table. If both the env
|
||||
var and the config request profiling, `json` wins. Example headline:
|
||||
|
||||
```text
|
||||
Slowest phase: reconciliation (2m 05s, 48% of wall time). 2 work units (1 failed), ~88% model generation vs ~12% tools.
|
||||
```
|
||||
|
||||
Work units run serially by default (`ingest.workUnits.maxConcurrency` is `1`);
|
||||
raise it in `ktx.yaml` if the profile shows the run is bound by serialized
|
||||
work-unit agent loops. If the provider reports an LLM rate limit, **ktx** shows
|
||||
a transient wait message and temporarily reduces effective work-unit concurrency
|
||||
according to `ingest.rateLimit`.
|
||||
|
||||
## Common errors
|
||||
|
||||
| Error | Cause | Recovery |
|
||||
|-------|-------|----------|
|
||||
| Connection not configured | The connection id is not present in `ktx.yaml` | Add the connection with `ktx setup` or update `ktx.yaml` |
|
||||
| Enrichment is not configured | Database ingest needs a model, embeddings, and scan-enrichment configuration | Run `ktx setup` to configure a model and embeddings |
|
||||
| Query history is unsupported | The selected database driver does not support query history | Run ingest without query-history flags |
|
||||
| Deep readiness is missing | `--deep` or query history needs model, embedding, and scan-enrichment configuration | Run `ktx setup` or rerun with `--fast` |
|
||||
| Query history is unsupported | The selected database driver does not support query history | Run fast ingest without query-history flags |
|
||||
| Python runtime is missing | The selected ingest target needs runtime-backed SQL analysis or source parsing | Accept the interactive prompt, rerun with `--yes`, or run the suggested `ktx admin runtime install` command |
|
||||
| Context-source options were ignored | Query-history flags were supplied for a context-source connection | Omit database-only flags when ingesting context-source connections |
|
||||
| Context-source options were ignored | Depth and query-history flags were supplied for a context-source connection | Omit database-only flags when ingesting context-source connections |
|
||||
| Text ingest stops early | `--fail-fast` was used and one item failed | Fix the failed item or rerun without `--fail-fast` to collect all failures |
|
||||
|
|
|
|||
|
|
@ -51,9 +51,8 @@ prompts.
|
|||
|
||||
| Flag | Description |
|
||||
|------|-------------|
|
||||
| `--llm-backend <backend>` | LLM backend: `anthropic`, `vertex`, `claude-code`, or `codex` |
|
||||
| `--llm-backend <backend>` | LLM backend: `anthropic`, `vertex`, or `claude-code` |
|
||||
| `--llm-backend claude-code` | Use the local Claude Code session for **ktx** LLM calls |
|
||||
| `--llm-backend codex` | Use local Codex authentication for **ktx** LLM calls |
|
||||
| `--llm-model <model>` | LLM model ID or backend model alias to validate and save |
|
||||
| `--anthropic-api-key-env <name>` | Environment variable containing the Anthropic API key |
|
||||
| `--anthropic-api-key-file <path>` | File containing the Anthropic API key |
|
||||
|
|
@ -63,14 +62,9 @@ prompts.
|
|||
|
||||
Choose only one Anthropic credential source. Anthropic credential flags are only
|
||||
valid with the Anthropic backend; Vertex flags are only valid with the Vertex
|
||||
backend. The `claude-code` and `codex` backends use local authentication instead
|
||||
backend. The `claude-code` backend uses local Claude Code authentication instead
|
||||
of Anthropic API key or Vertex flags. For Claude Code, `--llm-model` accepts
|
||||
`sonnet`, `opus`, `haiku`, or a full Claude model ID. For Codex, `--llm-model`
|
||||
accepts `codex`, `default`, or a `gpt-*` / `codex-*` model ID such as
|
||||
`gpt-5.5`; any other value is rejected before the auth probe. Run `codex` to
|
||||
see the models available to your login, and pick a `gpt-*` / `codex-*` id from
|
||||
that list. Note that `*-codex` API-billing model IDs (for example
|
||||
`gpt-5.3-codex`) are not available to ChatGPT-subscription logins.
|
||||
`sonnet`, `opus`, `haiku`, or a full Claude model ID.
|
||||
|
||||
### Embeddings
|
||||
|
||||
|
|
@ -137,8 +131,8 @@ BigQuery; and `databases` for ClickHouse.
|
|||
Query history setup is supported for Postgres, BigQuery, and Snowflake. The
|
||||
window flag applies to BigQuery and Snowflake; Postgres reads the current
|
||||
`pg_stat_statements` aggregate data instead of a time-windowed history table.
|
||||
Later `ktx ingest` runs build enriched context and need a configured model and
|
||||
embeddings, including when query history is enabled.
|
||||
Enabling query history makes deep ingest readiness matter for later
|
||||
`ktx ingest` runs.
|
||||
|
||||
When query history is enabled for PostgreSQL, Snowflake, or BigQuery,
|
||||
`ktx setup` runs a non-blocking readiness probe after the connection test
|
||||
|
|
@ -148,13 +142,6 @@ fix the prerequisite. If the later schema-context build also fails, interactive
|
|||
setup offers **Disable query history and retry** so you can finish database
|
||||
setup with `connections.<id>.context.queryHistory.enabled: false`.
|
||||
|
||||
After the schema scan completes, setup can derive query-history service-account
|
||||
filters from in-scope history. If **ktx** finds clear operational roles, it
|
||||
prints each proposed exclusion with a reason and writes
|
||||
`connections.<id>.context.queryHistory.filters.serviceAccounts` only when you
|
||||
apply the proposal. In non-interactive setup with `--yes`, the proposal is
|
||||
applied automatically. Existing `serviceAccounts` blocks are never overwritten.
|
||||
|
||||
For BigQuery, the remediation tells you to grant `roles/bigquery.resourceViewer`
|
||||
on the BigQuery project, or grant a custom role that contains
|
||||
`bigquery.jobs.listAll`.
|
||||
|
|
@ -173,9 +160,9 @@ sources. This is equivalent to passing `--skip-sources` in scripted setup.
|
|||
| `--source-git-url <url>` | Git URL for dbt, MetricFlow, or LookML |
|
||||
| `--source-branch <branch>` | Git branch for context-source setup |
|
||||
| `--source-subpath <path>` | Repo subpath for context-source setup |
|
||||
| `--source-auth-token-ref <ref>` | `env:` or `file:` credential reference for source repo auth or Notion integration token |
|
||||
| `--source-auth-token-ref <ref>` | `env:` or `file:` credential reference for source repo auth |
|
||||
| `--source-url <url>` | Source service URL for Metabase or Looker |
|
||||
| `--source-api-key-ref <ref>` | `env:` or `file:` API key reference for Metabase |
|
||||
| `--source-api-key-ref <ref>` | `env:` or `file:` API key reference for Metabase or Notion |
|
||||
| `--source-client-id <id>` | Looker client id |
|
||||
| `--source-client-secret-ref <ref>` | `env:` or `file:` Looker client secret reference |
|
||||
| `--source-warehouse-connection-id <id>` | Warehouse connection id used for context-source mapping |
|
||||
|
|
@ -204,17 +191,6 @@ ktx setup \
|
|||
--llm-backend claude-code \
|
||||
--llm-model opus
|
||||
|
||||
# Configure **ktx** to use local Codex authentication for LLM work
|
||||
ktx setup --llm-backend codex --llm-model gpt-5.5 --no-input
|
||||
```
|
||||
|
||||
When you choose `--llm-backend codex`, setup prints a warning if the public
|
||||
Codex SDK and CLI surface cannot prove full Claude-Code-style isolation. The
|
||||
backend restricts **ktx** runtime MCP tools to each run, but Codex may still
|
||||
load user Codex config and built-in command execution or read-only file
|
||||
capabilities.
|
||||
|
||||
```bash
|
||||
# Script a Postgres connection that reads its URL from the environment
|
||||
ktx setup \
|
||||
--project-dir ./analytics \
|
||||
|
|
@ -245,14 +221,6 @@ ktx setup \
|
|||
--source-warehouse-connection-id warehouse \
|
||||
--metabase-database-id 1
|
||||
|
||||
# Add a Notion source that crawls selected root pages
|
||||
ktx setup \
|
||||
--source notion \
|
||||
--source-connection-id notion-main \
|
||||
--source-auth-token-ref env:NOTION_TOKEN \
|
||||
--notion-crawl-mode selected_roots \
|
||||
--notion-root-page-id abc123def456
|
||||
|
||||
# Install project-scoped agent integration for Codex
|
||||
ktx setup --agents --target codex
|
||||
```
|
||||
|
|
|
|||
|
|
@ -11,16 +11,13 @@ the vocabulary agents use to generate correct SQL.
|
|||
|
||||
```bash
|
||||
ktx sl [options] [query...] # list (bare) or search (with query)
|
||||
ktx sl read <sourceName>
|
||||
ktx sl validate <sourceName>
|
||||
ktx sl validate <sourceName> [options]
|
||||
ktx sl query [options]
|
||||
```
|
||||
|
||||
- Bare `ktx sl` lists semantic sources.
|
||||
- `ktx sl <query...>` searches semantic sources. Multi-word queries are joined
|
||||
with a space.
|
||||
- `ktx sl read <sourceName>` prints the YAML for one source. Add
|
||||
`--connection-id` only when the source name exists in multiple connections.
|
||||
- `ktx sl <query...>` searches semantic sources (multi-word queries are
|
||||
joined with a space).
|
||||
- `ktx sl validate` and `ktx sl query` remain as explicit subcommands.
|
||||
|
||||
## Subcommands
|
||||
|
|
@ -29,7 +26,6 @@ ktx sl query [options]
|
|||
|-----------|-------------|
|
||||
| (none, no query) | List semantic sources |
|
||||
| (none, with query) | Search semantic sources |
|
||||
| `read <sourceName>` | Print the YAML for one semantic source |
|
||||
| `validate <sourceName>` | Validate a semantic source against the database schema |
|
||||
| `query` | Compile or execute a semantic query |
|
||||
|
||||
|
|
@ -44,23 +40,17 @@ ktx sl query [options]
|
|||
| `--output <mode>` | Output mode: `pretty` (default in TTY), `plain` (TSV), or `json` | `pretty` |
|
||||
| `--json` | Shortcut for `--output=json` (overrides `--output`) | `false` |
|
||||
|
||||
### `sl read`
|
||||
|
||||
| Flag | Description | Default |
|
||||
|------|-------------|---------|
|
||||
| `--connection-id <id>` | Optional **ktx** connection id for disambiguation | - |
|
||||
|
||||
### `sl validate`
|
||||
|
||||
| Flag | Description | Default |
|
||||
|------|-------------|---------|
|
||||
| `--connection-id <id>` | Optional **ktx** connection id for disambiguation | - |
|
||||
| `--connection-id <id>` | **ktx** connection id (required) | - |
|
||||
|
||||
### `sl query`
|
||||
|
||||
| Flag | Description | Default |
|
||||
|------|-------------|---------|
|
||||
| `--connection-id <id>` | Required **ktx** connection id | - |
|
||||
| `--connection-id <id>` | **ktx** connection id | - |
|
||||
| `--query-file <path>` | JSON semantic query file | - |
|
||||
| `--measure <measure>` | Measure to query; repeatable (at least one required) | - |
|
||||
| `--dimension <dimension>` | Dimension to include; repeatable | - |
|
||||
|
|
@ -75,9 +65,8 @@ ktx sl query [options]
|
|||
| `--no-input` | Disable interactive managed runtime installation | - |
|
||||
| `--max-rows <n>` | Maximum rows to return when executing | - |
|
||||
|
||||
`sl query` requires `--connection-id` and at least one `--measure` unless
|
||||
`--query-file` is set. `--query-file` must point to a JSON semantic query
|
||||
object.
|
||||
`sl query` requires at least one `--measure` unless `--query-file` is set.
|
||||
`--query-file` should point to a JSON semantic query object.
|
||||
|
||||
## Examples
|
||||
|
||||
|
|
@ -94,16 +83,7 @@ ktx sl --json
|
|||
# Search sources as JSON
|
||||
ktx sl "revenue" --json
|
||||
|
||||
# Print the YAML for a source name that is unique across connections
|
||||
ktx sl read orders
|
||||
|
||||
# Print the YAML for a source name that exists in multiple connections
|
||||
ktx sl --connection-id my-warehouse read orders
|
||||
|
||||
# Validate a source name that is unique across connections
|
||||
ktx sl validate orders
|
||||
|
||||
# Validate a source name that exists in multiple connections
|
||||
# Validate a source against the live schema
|
||||
ktx sl validate orders --connection-id my-warehouse
|
||||
|
||||
# Compile a query and view the generated SQL
|
||||
|
|
@ -164,12 +144,6 @@ shows `#1`, `#2`, and later rank badges for the displayed results. Plain and
|
|||
JSON output keep the raw `score` value, which is a ranking score rather than a
|
||||
percentage.
|
||||
|
||||
`ktx sl read <sourceName>` prints the source YAML directly to stdout when the
|
||||
source name is unique across connections. If the name exists in multiple
|
||||
connections, rerun the command with `--connection-id <id>`. The command does
|
||||
not wrap output in pretty, plain, or JSON formatting, so it can be piped to
|
||||
other tools.
|
||||
|
||||
```json
|
||||
{
|
||||
"sql": "SELECT orders.status, SUM(orders.total_amount) AS total_revenue FROM public.orders GROUP BY orders.status",
|
||||
|
|
@ -186,8 +160,7 @@ other tools.
|
|||
|
||||
| Error | Cause | Recovery |
|
||||
|-------|-------|----------|
|
||||
| Source not found | Source name or connection id is wrong | Run `ktx sl <query>` or `ktx sl --connection-id <id>` to find the exact source name, then retry `ktx sl read <sourceName>` or `ktx sl validate <sourceName>` |
|
||||
| Source name is ambiguous | The same source name exists in multiple connections | Rerun with `--connection-id <id>` from the error message |
|
||||
| Source not found | Source name or connection id is wrong | Run `ktx sl --json` and retry with an exact source name and connection id |
|
||||
| Validation fails | YAML references missing columns, invalid joins, or invalid SQL expressions | Fix the source YAML and rerun `ktx sl validate` |
|
||||
| Query compile fails | Measure, dimension, filter, or segment name is invalid | Search sources with `ktx sl <query>`, inspect the source YAML in your project files, then retry using declared fields |
|
||||
| Execution returns too many rows | `--max-rows` is missing or too high | Add `--max-rows` with a bounded value before executing |
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ ktx status [options]
|
|||
| `--json` | Print JSON output | `false` |
|
||||
| `-v`, `--verbose` | Show every check, including passing ones | `false` |
|
||||
| `--validate` | Only validate the `ktx.yaml` schema; skip readiness checks | `false` |
|
||||
| `--fast` | Skip checks that require external communication (query-history readiness probes, Claude Code auth probe, and Codex auth probe) | `false` |
|
||||
| `--fast` | Skip checks that require external communication (query-history readiness probes and Claude Code auth probe) | `false` |
|
||||
| `--no-input` | Disable interactive terminal input | - |
|
||||
|
||||
## Examples
|
||||
|
|
@ -39,7 +39,7 @@ ktx status --verbose
|
|||
# Validate ktx.yaml without running readiness checks
|
||||
ktx status --validate
|
||||
|
||||
# Skip slow probes (query-history readiness, Claude Code auth, Codex auth)
|
||||
# Skip slow probes (query-history readiness, Claude Code auth)
|
||||
ktx status --fast
|
||||
|
||||
# Check a project from another directory
|
||||
|
|
@ -57,16 +57,6 @@ flow, then rerun `ktx status`. Use `--fast` to skip this probe (useful in CI
|
|||
or offline contexts); skipped checks render as `-` and carry
|
||||
`"status": "skipped"` in JSON output.
|
||||
|
||||
For `llm.provider.backend: codex`, `ktx status` runs a minimal non-interactive
|
||||
Codex request. If the probe fails, authenticate Codex locally with the Codex CLI
|
||||
and verify the Codex CLI installation.
|
||||
|
||||
When `llm.provider.backend: codex` is configured, `ktx status` also prints a
|
||||
warning when the installed public Codex SDK and CLI surface cannot prove full
|
||||
Claude-Code-style isolation. The warning does not block authenticated Codex
|
||||
usage, but it marks the project status as partial so you can make an explicit
|
||||
runtime-isolation decision.
|
||||
|
||||
A `Local data` section summarises what the project has accumulated locally:
|
||||
ingest run counts, last completed timestamp per connection, knowledge page
|
||||
counts by scope, semantic-layer source and dictionary value counts, and the
|
||||
|
|
|
|||
|
|
@ -1,24 +1,21 @@
|
|||
---
|
||||
title: "ktx wiki"
|
||||
description: "List, search, or read wiki pages."
|
||||
description: "List or search wiki pages."
|
||||
---
|
||||
|
||||
List, search, and read wiki pages in your **ktx** project. Wiki pages are
|
||||
Markdown documents that capture business definitions, rules, and gotchas.
|
||||
Agents search them for context when answering questions about your data.
|
||||
List and search wiki pages in your **ktx** project. Wiki pages are Markdown
|
||||
documents that capture business definitions, rules, and gotchas. Agents search
|
||||
them for context when answering questions about your data.
|
||||
|
||||
## Command signature
|
||||
|
||||
```bash
|
||||
ktx wiki [options] [query...] # list (bare) or search (with query)
|
||||
ktx wiki read <key>
|
||||
ktx wiki [options] [query...]
|
||||
```
|
||||
|
||||
- Bare `ktx wiki` lists local wiki pages.
|
||||
- `ktx wiki <query...>` searches local wiki pages. Multi-word queries are
|
||||
joined with a space.
|
||||
- `ktx wiki read <key>` prints the whole Markdown file for one wiki page,
|
||||
including YAML frontmatter.
|
||||
- `ktx wiki <query...>` searches local wiki pages (multi-word queries are
|
||||
joined with a space).
|
||||
|
||||
Edit the Markdown files under `wiki/` directly, or ingest source content with
|
||||
`ktx ingest`, when you need to add or update wiki knowledge.
|
||||
|
|
@ -53,9 +50,6 @@ ktx wiki "monthly recurring revenue"
|
|||
# Search wiki pages as JSON
|
||||
ktx wiki "monthly recurring revenue" --json --limit 10
|
||||
|
||||
# Print the exact Markdown file for a known page key
|
||||
ktx wiki read revenue-definitions
|
||||
|
||||
# Print search results as TSV
|
||||
ktx wiki "monthly recurring revenue" --output plain
|
||||
|
||||
|
|
@ -68,10 +62,8 @@ ktx --debug wiki "monthly recurring revenue" --json
|
|||
Wiki commands print clack-style pretty output in a TTY and TSV-style plain
|
||||
output when requested. JSON output wraps the items with a command metadata
|
||||
envelope. Search results include `matchReasons` and `lanes` metadata so you can
|
||||
see whether lexical, token, or semantic search contributed to the ranking. Use
|
||||
`ktx wiki read <key>` when you need the full page contents. Read output is the
|
||||
exact Markdown file stored on disk, including YAML frontmatter, and is not
|
||||
wrapped in pretty, plain, or JSON formatting.
|
||||
see whether lexical, token, or semantic search contributed to the ranking. Open
|
||||
the matching Markdown files directly when you need the full page contents.
|
||||
Pretty search output shows `#1`, `#2`, and later rank badges for the displayed
|
||||
results. Plain and JSON output keep the raw `score` value, which is a ranking
|
||||
score rather than a percentage.
|
||||
|
|
@ -129,4 +121,4 @@ stays machine-readable:
|
|||
| Error | Cause | Recovery |
|
||||
|-------|-------|----------|
|
||||
| Search returns no results | The query terms do not match summaries, tags, or content, and the semantic lane is unavailable or has no positive matches | Run with `--debug`, check the semantic lane status, retry with business synonyms, then create a page if the knowledge is missing |
|
||||
| A page is missing | No Markdown file exists for that business context or `ktx wiki read <key>` used the wrong key | Run `ktx wiki <query>` to find the page key, then retry `ktx wiki read <key>` |
|
||||
| A page is missing | No Markdown file exists for that business context | Add a file under `wiki/` or run `ktx ingest <connectionId>` |
|
||||
|
|
|
|||
|
|
@ -36,11 +36,9 @@ ktx
|
|||
wiki
|
||||
list
|
||||
search <query>
|
||||
read <key>
|
||||
sl
|
||||
list
|
||||
search <query>
|
||||
read <sourceName>
|
||||
validate <sourceName>
|
||||
query
|
||||
sql
|
||||
|
|
@ -59,7 +57,6 @@ ktx
|
|||
stop
|
||||
status
|
||||
reindex
|
||||
completion <shell>
|
||||
```
|
||||
|
||||
The public context-build entrypoint is `ktx ingest [connectionId]` or
|
||||
|
|
@ -74,44 +71,6 @@ The public context-build entrypoint is `ktx ingest [connectionId]` or
|
|||
| `-v`, `--version` | Show the CLI package name and version. |
|
||||
| `-h`, `--help` | Show help for the current command. |
|
||||
|
||||
## Update notices
|
||||
|
||||
> **Note:** The update notifier writes only to stderr and keeps command stdout
|
||||
> unchanged.
|
||||
|
||||
When a newer package is available on your installed release channel, `ktx`
|
||||
prints a short notice after the command finishes:
|
||||
|
||||
```text
|
||||
↑ Update available: ktx 0.9.0 → 0.10.0
|
||||
npm i -g @kaelio/ktx
|
||||
```
|
||||
|
||||
Stable installs compare against the npm `latest` dist-tag.
|
||||
Release-candidate installs compare against the `next` dist-tag and show:
|
||||
|
||||
```text
|
||||
npm i -g @kaelio/ktx@next
|
||||
```
|
||||
|
||||
The check is skipped for JSON output, CI, non-TTY stdout, and hidden completion
|
||||
commands. To opt out explicitly, set any of these environment variables:
|
||||
|
||||
```bash
|
||||
KTX_NO_UPDATE_CHECK=1
|
||||
NO_UPDATE_NOTIFIER=1
|
||||
DO_NOT_TRACK=1
|
||||
```
|
||||
|
||||
The `ktx` CLI prints one npm command because globally installed binaries don't
|
||||
expose a reliable runtime package-manager signal. If you prefer another global
|
||||
package manager, use the equivalent command:
|
||||
|
||||
```bash
|
||||
pnpm add -g @kaelio/ktx
|
||||
yarn global add @kaelio/ktx
|
||||
```
|
||||
|
||||
## Project resolution
|
||||
|
||||
Most commands are project-aware. Pass `--project-dir <path>` when scripting or
|
||||
|
|
@ -138,10 +97,6 @@ ktx ingest
|
|||
ktx sl "revenue"
|
||||
ktx wiki "revenue recognition"
|
||||
|
||||
# Print a known wiki page or semantic source
|
||||
ktx wiki read revenue-definitions
|
||||
ktx sl --connection-id warehouse read orders
|
||||
|
||||
# Execute read-only SQL
|
||||
ktx sql --connection warehouse "select count(*) from public.orders"
|
||||
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@
|
|||
"ktx-wiki",
|
||||
"ktx-status",
|
||||
"ktx-mcp",
|
||||
"ktx-admin",
|
||||
"ktx-completion"
|
||||
"ktx-admin"
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,15 +1,12 @@
|
|||
---
|
||||
title: Telemetry
|
||||
description: Understand what usage telemetry ktx collects and how to opt out.
|
||||
description: Understand what anonymous usage telemetry ktx collects and how to opt out.
|
||||
---
|
||||
|
||||
**ktx** collects aggregated usage telemetry so maintainers can see
|
||||
which commands work, where setup fails, and which parts of the data-agent
|
||||
workflow need improvement. Telemetry is opt-out: it turns on the first time you
|
||||
run **ktx** in any way — an interactive command, a script, or an
|
||||
agent-launched MCP server — and prints a one-time notice (to the terminal when
|
||||
there is one, otherwise to standard error). It stays disabled in CI and whenever
|
||||
an opt-out is set.
|
||||
**ktx** collects anonymous, aggregated usage telemetry from interactive CLI
|
||||
runs so maintainers can see which commands work, where setup fails, and which
|
||||
parts of the data-agent workflow need improvement. Telemetry is opt-out and
|
||||
disabled automatically in CI and non-interactive runs.
|
||||
|
||||
## Opt out
|
||||
|
||||
|
|
@ -20,58 +17,23 @@ Use any of these mechanisms to disable telemetry:
|
|||
| `export KTX_TELEMETRY_DISABLED=1` | Disables telemetry for the shell and child processes |
|
||||
| `export DO_NOT_TRACK=1` | Standard do-not-track environment variable |
|
||||
| `CI=1` | Automatic in CI |
|
||||
| Edit `~/.ktx/telemetry.json` and set `"enabled": false` | Persistent for the machine, including the MCP server |
|
||||
| Non-TTY output | Automatic for pipes and scripts |
|
||||
| Edit `~/.ktx/telemetry.json` and set `"enabled": false` | Persistent for the machine |
|
||||
|
||||
## What we collect
|
||||
|
||||
High-level signals: which commands run, how long they take, whether they
|
||||
High-level signals only: which commands run, how long they take, whether they
|
||||
succeed or fail, and basic environment metadata (CLI version, Node version, OS
|
||||
platform). When an operation fails, we also include diagnostic detail about the
|
||||
error so we can debug it. For project-level analysis, **ktx** sends a salted
|
||||
hash of the project directory to group events.
|
||||
|
||||
When an agent reaches **ktx** through MCP, we also record the connecting client
|
||||
tool's self-reported name and version (for example Claude Desktop, Cursor, or
|
||||
Cline) so we can see which agents people use **ktx** with. That describes the
|
||||
tool, never you or your data.
|
||||
platform). For project-level analysis, **ktx** sends a salted hash of the
|
||||
project directory — never the raw path.
|
||||
|
||||
## What we never collect
|
||||
|
||||
We build telemetry around counts and coarse signals, not the contents of your
|
||||
data or configuration. We don't deliberately collect your `ktx.yaml`, query
|
||||
results, passwords, API keys, or access tokens.
|
||||
|
||||
The one place environment-specific text can appear is failure diagnostics: when
|
||||
an operation errors, the detail we record is the error as your tools reported
|
||||
it, which can include identifiers from your setup. If you'd rather send nothing
|
||||
at all, turn telemetry off using any of the options above.
|
||||
|
||||
## Error reports
|
||||
|
||||
When telemetry is enabled, **ktx** sends PostHog Error Tracking `$exception`
|
||||
events for CLI and daemon exceptions. Error reports help group crashes and
|
||||
handled failures into PostHog issues.
|
||||
|
||||
Error reports can include:
|
||||
|
||||
- Stack frames, including function names, local file paths, line numbers, and
|
||||
SDK-provided source context.
|
||||
- Error class names and raw error messages.
|
||||
- Cause chains when the runtime exposes them.
|
||||
- `source`, `handled`, and `fatal` diagnostic fields.
|
||||
- Runtime version, OS, architecture, and CI fields.
|
||||
- The hashed `projectId` when **ktx** knows the project.
|
||||
|
||||
Error reports never intentionally include:
|
||||
|
||||
- Secrets, credentials, API keys, tokens, cookies, signed URLs, or auth headers.
|
||||
- Database URLs, connection strings, DSNs, raw argv, or raw environment values.
|
||||
- SQL text, schema names, table names, or column names as explicit payload
|
||||
properties.
|
||||
- Customer row data.
|
||||
- User prompt text or raw MCP arguments.
|
||||
|
||||
The same opt-out controls listed above disable error reports.
|
||||
- File paths, hostnames, environment variable values, or command arguments
|
||||
- `ktx.yaml` contents, connection passwords, API keys, or tokens
|
||||
- Schema names, table names, column names, SQL text, or query results
|
||||
- Error messages or stack traces
|
||||
- Git remote URLs, Git user email, OS user, or hostname
|
||||
|
||||
## Storage and retention
|
||||
|
||||
|
|
|
|||
|
|
@ -66,9 +66,8 @@ read, how to think, and where to put the results.
|
|||
## Minimal config
|
||||
|
||||
A working `ktx.yaml` needs one entry in `connections`. Everything else accepts
|
||||
defaults. The example below registers a local Postgres connection; building
|
||||
context with `ktx ingest warehouse` also needs a model and embeddings, which
|
||||
`ktx setup` configures.
|
||||
defaults. The example below is enough for `ktx ingest warehouse` to run a fast
|
||||
schema scan against a local Postgres.
|
||||
|
||||
```yaml
|
||||
connections:
|
||||
|
|
@ -124,7 +123,7 @@ context-source drivers share the map.
|
|||
|
||||
Warehouse connections are open objects: the listed fields are validated, and
|
||||
any other field is preserved and passed through to the connector. Use
|
||||
`enabled_tables` to scope ingest to a specific list of
|
||||
`enabled_tables` to scope deep ingest to a specific list of
|
||||
`schema.table` names - useful for smoke tests.
|
||||
|
||||
```yaml
|
||||
|
|
@ -179,22 +178,9 @@ connections:
|
|||
context:
|
||||
queryHistory:
|
||||
enabled: true
|
||||
enabledSchemas:
|
||||
- orbit_raw
|
||||
- orbit_analytics
|
||||
minExecutions: 5
|
||||
```
|
||||
|
||||
- `enabledSchemas`: Optional list of schema or dataset names that query-history
|
||||
ingest may mine. Omit it to let **ktx** derive the modeled schema floor from
|
||||
the connection and semantic-layer sources. Use `["*"]` to disable the floor
|
||||
for discovery runs.
|
||||
- `filters.serviceAccounts`: Optional service-account filter block. During
|
||||
setup, when query history is enabled and no service-account block already
|
||||
exists, **ktx** can propose exact role patterns such as `^svc_loader$` from
|
||||
observed in-scope query history. The block uses `mode: exclude` and remains
|
||||
hand-editable.
|
||||
|
||||
### Metabase
|
||||
|
||||
```yaml
|
||||
|
|
@ -389,23 +375,13 @@ llm:
|
|||
|
||||
| Field | Type | Default | Purpose |
|
||||
|-------|------|---------|---------|
|
||||
| `provider.backend` | `none` \| `anthropic` \| `vertex` \| `gateway` \| `claude-code` \| `codex` | `none` | Selected backend. `none` disables LLM features. `claude-code` uses the local Claude Code session and needs no API key. `codex` uses local Codex authentication and needs no API key. |
|
||||
| `provider.backend` | `none` \| `anthropic` \| `vertex` \| `gateway` \| `claude-code` | `none` | Selected backend. `none` disables LLM features. `claude-code` uses the local Claude Code session and needs no API key. |
|
||||
| `provider.anthropic.api_key` | `string` | - | Anthropic API key. Required when `backend: anthropic`. Accepts `env:` or `file:` references. |
|
||||
| `provider.anthropic.base_url` | `string` | - | Override the Anthropic API base URL (proxy, self-hosted gateway). |
|
||||
| `provider.gateway.api_key` / `base_url` | `string` | - | Credentials for an AI Gateway provider. Required when `backend: gateway`. |
|
||||
| `provider.vertex.project` | `string` | - | Google Cloud project ID hosting the Vertex AI endpoint. |
|
||||
| `provider.vertex.location` | `string` | - | Vertex AI region (for example `us-east5`). Required when the `vertex` block is present. |
|
||||
|
||||
Use `codex` when local Codex authentication should power **ktx** LLM work:
|
||||
|
||||
```yaml
|
||||
llm:
|
||||
provider:
|
||||
backend: codex
|
||||
models:
|
||||
default: gpt-5.5
|
||||
```
|
||||
|
||||
### Model roles
|
||||
|
||||
`models` overrides the per-role model. Keys are fixed; values are
|
||||
|
|
@ -452,16 +428,6 @@ ingest:
|
|||
stepBudget: 40
|
||||
maxConcurrency: 2
|
||||
failureMode: continue
|
||||
rateLimit:
|
||||
enabled: true
|
||||
throttleThreshold: 0.8
|
||||
minConcurrencyUnderPressure: 1
|
||||
maxWaitMs: 600000
|
||||
retry:
|
||||
maxAttempts: 6
|
||||
baseDelayMs: 1000
|
||||
maxDelayMs: 60000
|
||||
jitter: true
|
||||
```
|
||||
|
||||
### Adapters
|
||||
|
|
@ -508,24 +474,6 @@ handles failures.
|
|||
| `workUnits.maxConcurrency` | `int > 0` | `1` | How many work units run in parallel. |
|
||||
| `workUnits.failureMode` | `abort` \| `continue` | `continue` | `abort` stops the whole ingest run on the first failure; `continue` records it and keeps going. |
|
||||
|
||||
### Rate limits
|
||||
|
||||
`rateLimit` controls provider-neutral pacing for LLM calls during ingest. When a
|
||||
provider reports a subscription window, retry-after delay, or HTTP 429,
|
||||
**ktx** pauses new work-unit model calls, shows a transient wait in the CLI,
|
||||
and reduces work-unit concurrency while the provider is under pressure.
|
||||
|
||||
| Field | Type | Default | Purpose |
|
||||
|-------|------|---------|---------|
|
||||
| `rateLimit.enabled` | `boolean` | `true` | Master switch for ingest LLM rate-limit pacing and visible waits. |
|
||||
| `rateLimit.throttleThreshold` | `number between 0 and 1` | `0.8` | Fraction of a known provider window at which **ktx** starts reducing concurrency. |
|
||||
| `rateLimit.minConcurrencyUnderPressure` | `int > 0` | `1` | Effective work-unit concurrency while a provider is under rate-limit pressure. |
|
||||
| `rateLimit.maxWaitMs` | `int > 0` | unset | Caps how long a single provider-reset wait can last. This bounds each wait, not the whole run: after a capped wait elapses **ktx** retries and may pause again. Omit to wait until the provider's reset time. |
|
||||
| `rateLimit.retry.maxAttempts` | `int > 0` | `6` | Maximum attempts for a single rate-limited LLM call before the failure surfaces (counts the first try). Also bounds how far opaque backoff grows for responses without a reset time or retry-after value. |
|
||||
| `rateLimit.retry.baseDelayMs` | `int > 0` | `1000` | Initial opaque retry delay in milliseconds. |
|
||||
| `rateLimit.retry.maxDelayMs` | `int > 0` | `60000` | Maximum opaque retry delay in milliseconds. |
|
||||
| `rateLimit.retry.jitter` | `boolean` | `true` | Add jitter to opaque retry delays. |
|
||||
|
||||
## `scan`
|
||||
|
||||
`scan` configures how schema-level inputs become structured context:
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ description: ktx is an open-source, self-improving context layer for data agents
|
|||
---
|
||||
|
||||
import { ProductMechanics } from "@/components/product-mechanics";
|
||||
import { ProductRuntime } from "@/components/product-runtime";
|
||||
|
||||
<div className="not-prose mb-10">
|
||||
<div>
|
||||
|
|
@ -60,8 +59,6 @@ serves that context to agents at runtime.
|
|||
|
||||
<ProductMechanics />
|
||||
|
||||
<ProductRuntime />
|
||||
|
||||
## Use it for
|
||||
|
||||
Use **ktx** when agents need more than raw database access. Agents can search wiki
|
||||
|
|
|
|||
|
|
@ -30,18 +30,17 @@ If you're a coding assistant choosing a docs route, start with the
|
|||
</div>
|
||||
<div className="mt-2.5 text-base leading-relaxed text-fd-foreground">
|
||||
Try **ktx** against a real data stack - Postgres, dbt, Metabase, and Notion
|
||||
pre-loaded with the Orbit demo corpus. Hit **copy agent setup** on the page
|
||||
for a one-shot prompt that has an agent install the full four-source demo,
|
||||
or grab the raw credentials to paste into `ktx setup` yourself.
|
||||
pre-loaded with the Orbit demo corpus. The page lists demo credentials
|
||||
you can paste straight into `ktx setup`.
|
||||
</div>
|
||||
<a
|
||||
href="https://www.kaelio.com/start"
|
||||
href="https://kaelio.com/start"
|
||||
className="group mt-5 inline-flex items-center gap-2 rounded-full px-4 py-2.5 text-sm font-semibold text-white no-underline shadow-[inset_0_1px_0_rgba(255,255,255,0.35),0_2px_4px_rgba(255,138,77,0.2),0_10px_24px_-8px_rgba(255,138,77,0.55)] transition-all duration-200 hover:-translate-y-0.5 hover:shadow-[inset_0_1px_0_rgba(255,255,255,0.4),0_3px_6px_rgba(255,138,77,0.28),0_16px_30px_-8px_rgba(255,138,77,0.65)]"
|
||||
style={{
|
||||
background: 'linear-gradient(180deg, #ff9d63 0%, #f97316 100%)',
|
||||
}}
|
||||
>
|
||||
Get demo credentials at www.kaelio.com/start
|
||||
Get demo credentials at kaelio.com/start
|
||||
<svg
|
||||
width="14"
|
||||
height="14"
|
||||
|
|
@ -215,8 +214,8 @@ The wizard walks you through everything **ktx** needs in one pass:
|
|||
SQLite, PostgreSQL, MySQL, SQL Server, BigQuery, and Snowflake.
|
||||
5. **Context sources** - optionally adds dbt, MetricFlow, LookML, Looker,
|
||||
Metabase, or Notion. You can skip and add them later.
|
||||
6. **Build** - offers to run the first ingest so semantic sources and wiki
|
||||
pages are ready for agents. If you skip it, build later with `ktx ingest`.
|
||||
6. **Build** - runs the first ingest so semantic sources and wiki pages
|
||||
are ready for agents.
|
||||
7. **Agent integration** - installs project-local rules for Claude Code,
|
||||
Codex, Cursor, OpenCode, or universal `.agents`.
|
||||
|
||||
|
|
@ -236,7 +235,7 @@ Testing warehouse
|
|||
Connection test passed
|
||||
|
||||
Building schema context for warehouse
|
||||
Running database scan
|
||||
Running fast database ingest
|
||||
```
|
||||
|
||||
If setup exits early, rerun `ktx setup` in the same directory. **ktx** keeps
|
||||
|
|
@ -247,18 +246,6 @@ progress under `.ktx/setup/` and resumes from the remaining work.
|
|||
> resuming setup, connecting an agent, checking status, or exploring a
|
||||
> pre-built demo project.
|
||||
|
||||
When the wizard finishes, it states where you stand and the single next action:
|
||||
|
||||
- **Context built** - **ktx** confirms it is ready for agents and points you to
|
||||
open your coding agent and ask a data question.
|
||||
- **Build skipped** - **ktx** tells you setup is complete and that the only step
|
||||
left is to build context with `ktx ingest`.
|
||||
|
||||
Re-running `ktx setup` on an already-configured project goes straight to the
|
||||
remaining step - building context or connecting an agent - instead of
|
||||
re-asking every question. Once everything is ready, it confirms you are set
|
||||
rather than reopening the configuration menu.
|
||||
|
||||
## Verify
|
||||
|
||||
When setup finishes, check readiness:
|
||||
|
|
@ -280,41 +267,18 @@ Agent integration ready: yes (codex:project)
|
|||
|
||||
For a structured check inside scripts, use `ktx status --json`.
|
||||
|
||||
If you skipped the build, `ktx context built` shows `no`. Build it with
|
||||
`ktx ingest` - there is no need to re-run `ktx setup`.
|
||||
|
||||
When setup finishes building context, its final context check looks like:
|
||||
When setup builds deep context, its final context check looks like:
|
||||
|
||||
```text
|
||||
ktx context is ready for agents.
|
||||
|
||||
Databases:
|
||||
warehouse: database context complete
|
||||
warehouse: deep context complete
|
||||
|
||||
Context sources:
|
||||
dbt_main: memory update complete
|
||||
```
|
||||
|
||||
Before the build starts, **ktx** runs a live test for every connection the
|
||||
build depends on. A context build can take several minutes, so if any required
|
||||
connection is unreachable or misconfigured the build is blocked up front and
|
||||
**ktx** names the failing connection by id and connector type:
|
||||
|
||||
```text
|
||||
KTX cannot build context: a required connection failed its live test.
|
||||
|
||||
Failed connections:
|
||||
warehouse (postgres)
|
||||
|
||||
Each connection must be reachable before KTX builds context.
|
||||
Run `ktx connection test <id>` to see the error, fix the connection, then retry.
|
||||
```
|
||||
|
||||
Run `ktx connection test <connection-id>` to see the underlying error, fix the
|
||||
connection, then continue. In interactive setup you can retry without
|
||||
restarting; with `--no-input` the build exits non-zero and names the failing
|
||||
connection so scripts can stop early.
|
||||
|
||||
## Connect a coding agent
|
||||
|
||||
The setup wizard installs project-local agent rules in the last step. To
|
||||
|
|
@ -361,7 +325,7 @@ ktx setup \
|
|||
Then build context:
|
||||
|
||||
```bash
|
||||
ktx ingest warehouse
|
||||
ktx ingest warehouse --fast
|
||||
```
|
||||
|
||||
See [ktx setup](/docs/cli-reference/ktx-setup) for the full automation flag
|
||||
|
|
@ -374,8 +338,7 @@ surface.
|
|||
| `ktx: command not found` | Reinstall `@kaelio/ktx` and open a new shell |
|
||||
| Setup resumes the wrong project | Pass `--project-dir <path>` |
|
||||
| LLM or embeddings health check fails | Rerun setup and pick a different credential, model, or backend |
|
||||
| Database test fails | Use the setup recovery menu to retry or re-enter details; if it still fails, verify the same connection with the database's native client |
|
||||
| Context build blocked: a connection failed its live test | Run `ktx connection test <connection-id>` to see the error, fix the connection, then retry the build |
|
||||
| Database test fails | Verify the same connection with the database's native client, then rerun setup |
|
||||
| Agent integration is incomplete | Run `ktx setup --agents --target <target>` |
|
||||
|
||||
## Next steps
|
||||
|
|
|
|||
|
|
@ -24,9 +24,7 @@ external metadata can attach to known warehouse tables.
|
|||
|
||||
## Database ingest
|
||||
|
||||
Database ingest always builds enriched context: tables, columns, types,
|
||||
constraints, and row counts, plus AI-generated descriptions, embeddings, and
|
||||
relationship evidence.
|
||||
Database ingest records table, column, type, constraint, and row-count context.
|
||||
|
||||
```bash
|
||||
# Build one configured database connection
|
||||
|
|
@ -36,37 +34,37 @@ ktx ingest warehouse
|
|||
ktx ingest --all
|
||||
```
|
||||
|
||||
Enriched ingest needs a configured model and embeddings. Run `ktx setup` first;
|
||||
connections without that configuration fail before any work starts.
|
||||
Depth controls how much context **ktx** builds:
|
||||
|
||||
Local-auth backends keep provider credentials out of `ktx.yaml`:
|
||||
| Flag | Best for | What it does |
|
||||
|------|----------|--------------|
|
||||
| `--fast` | First setup, quick refreshes, CI smoke checks | Deterministic fast ingest with tables, columns, types, constraints, and row counts |
|
||||
| `--deep` | Agent-ready context for real analysis | Fast ingest plus deep enrichment with descriptions, embeddings, relationship evidence, and optional query history |
|
||||
|
||||
Examples:
|
||||
|
||||
```bash
|
||||
ktx setup --llm-backend claude-code --no-input
|
||||
ktx setup --llm-backend codex --llm-model gpt-5.5 --no-input
|
||||
ktx ingest warehouse --fast
|
||||
ktx ingest warehouse --deep
|
||||
ktx ingest --all --deep
|
||||
```
|
||||
|
||||
With `claude-code`, **ktx** agent loops can invoke only the **ktx** MCP tools
|
||||
for the current run. With `codex`, **ktx** restricts the temporary runtime MCP
|
||||
server to the current run's tool set, disables Codex web search, requests a
|
||||
read-only sandbox, and sets `approval_policy=never`. The public Codex SDK and
|
||||
CLI surface may still load user Codex config and built-in command execution or
|
||||
read-only file capabilities, so use `claude-code` for stricter runtime tool
|
||||
isolation.
|
||||
Deep ingest needs LLM and embedding readiness. Otherwise run `ktx setup` or use
|
||||
`--fast`.
|
||||
|
||||
With `claude-code`, **ktx** agent loops can invoke only the **ktx** MCP tools for the
|
||||
current run.
|
||||
|
||||
## Query history
|
||||
|
||||
PostgreSQL, BigQuery, and Snowflake can add query-history context: common joins,
|
||||
filters, redaction rules, high-usage templates, and service-account exclusions.
|
||||
When query history is enabled during setup, **ktx** reviews observed in-scope
|
||||
roles and can write exact `filters.serviceAccounts` patterns for operational
|
||||
traffic such as loader or refresh roles.
|
||||
filters, service-account patterns, redaction rules, and high-usage templates.
|
||||
|
||||
Enable it during setup, store it under `connections.<id>.context.queryHistory`,
|
||||
or request it for one run:
|
||||
|
||||
```bash
|
||||
ktx ingest warehouse --query-history
|
||||
ktx ingest warehouse --deep --query-history
|
||||
# Set the lookback window for BigQuery or Snowflake query history
|
||||
ktx ingest warehouse --query-history-window-days 30
|
||||
```
|
||||
|
|
@ -76,8 +74,8 @@ for one run.
|
|||
|
||||
## Relationship evidence
|
||||
|
||||
**ktx** scores relationship candidates during database ingest. The public CLI
|
||||
does not expose separate relationship review subcommands.
|
||||
**ktx** scores relationship candidates during supported deep database ingest. The
|
||||
public CLI does not expose separate relationship review subcommands.
|
||||
|
||||
## Context-source ingest
|
||||
|
||||
|
|
@ -161,7 +159,7 @@ After interactive setup:
|
|||
|
||||
```bash
|
||||
ktx status
|
||||
ktx ingest --all
|
||||
ktx ingest --all --deep
|
||||
ktx status
|
||||
```
|
||||
|
||||
|
|
@ -178,8 +176,8 @@ ktx wiki "revenue" --json --limit 10
|
|||
| Symptom | Likely cause | Recovery |
|
||||
|---------|--------------|----------|
|
||||
| Connection not configured | The connection id is missing from `ktx.yaml` | Add it with `ktx setup` |
|
||||
| Enrichment is not configured | LLM or embeddings are not setup-ready | Run `ktx setup` to configure a model and embeddings |
|
||||
| Query history is unsupported | The selected database driver does not expose query history | Run ingest without query-history flags |
|
||||
| Deep readiness is missing | LLM or embeddings are not setup-ready | Run `ktx setup`, or rerun with `--fast` |
|
||||
| Query history is unsupported | The selected database driver does not expose query history | Run fast ingest without query-history flags |
|
||||
| No connections configured | The project has no entries under `connections` | Run `ktx setup` and add a database or context-source connection |
|
||||
| Context-source flags have no effect | Query-history flags were supplied for a context-source connector | Use query-history flags only for database connections |
|
||||
| Context-source flags have no effect | Depth and query-history flags were supplied for a context-source connector | Use those flags only for database connections |
|
||||
| Text ingest stops early | `--fail-fast` stopped on the first failed item | Fix the item or rerun without `--fail-fast` |
|
||||
|
|
|
|||
|
|
@ -16,7 +16,6 @@ Set `llm.provider.backend` to one of these values:
|
|||
- `gateway`: Use AI Gateway-compatible Anthropic model ids.
|
||||
- `claude-code`: Use your local Claude Code session through the Claude Agent
|
||||
SDK. **ktx** strips provider-routing environment variables from child processes.
|
||||
- `codex`: Use your local Codex authentication through the Codex SDK.
|
||||
|
||||
## Claude Code
|
||||
|
||||
|
|
@ -48,42 +47,6 @@ model IDs are also accepted.
|
|||
metadata may still list host slash commands, skills, and subagents; **ktx** does not
|
||||
grant execution access to them.
|
||||
|
||||
## Codex backend
|
||||
|
||||
Use `codex` when you want **ktx** to run LLM-backed workflows through your
|
||||
local Codex authentication instead of a direct provider API key.
|
||||
|
||||
```yaml
|
||||
llm:
|
||||
provider:
|
||||
backend: codex
|
||||
models:
|
||||
default: gpt-5.5
|
||||
```
|
||||
|
||||
Configure it non-interactively:
|
||||
|
||||
```bash
|
||||
ktx setup --llm-backend codex --llm-model gpt-5.5 --no-input
|
||||
```
|
||||
|
||||
This is separate from Codex agent-client setup. `ktx setup --agents --target
|
||||
codex` installs instructions and MCP access for an end-user Codex session.
|
||||
`ktx setup --llm-backend codex` makes **ktx** itself execute ingest, scan
|
||||
enrichment, memory, and other LLM-backed work through Codex.
|
||||
|
||||
During runtime loops, **ktx** starts a temporary loopback MCP server for the
|
||||
current run, exposes only the tools passed to that run, asks Codex to use a
|
||||
read-only sandbox, sets `approval_policy=never`, auto-approves only those
|
||||
run-scoped MCP tools, and disables Codex web search.
|
||||
|
||||
Codex backend isolation is currently limited by the public Codex SDK and CLI
|
||||
surface. Codex may still load user Codex config and built-in command execution
|
||||
or read-only file capabilities. Use `llm.provider.backend: claude-code` when
|
||||
you need stricter Claude-Code-style runtime tool isolation, or remove host
|
||||
Codex MCP and tool config before running untrusted prompts through the `codex`
|
||||
backend.
|
||||
|
||||
## Prompt caching
|
||||
|
||||
`llm.promptCaching` has partial parity on `claude-code`. Status and doctor warn
|
||||
|
|
|
|||
|
|
@ -111,13 +111,12 @@ non-obvious terms.
|
|||
Agents can refresh context when the user asks them to:
|
||||
|
||||
```bash
|
||||
ktx ingest warehouse
|
||||
ktx ingest warehouse --fast
|
||||
ktx ingest
|
||||
ktx ingest --file docs/revenue-notes.md --connection-id warehouse
|
||||
```
|
||||
|
||||
Database ingest builds enriched context and requires a configured model and
|
||||
embeddings; run `ktx setup` first if they are not ready.
|
||||
Use `--deep` only when LLM and embedding setup is ready.
|
||||
|
||||
## Good agent behavior
|
||||
|
||||
|
|
|
|||
|
|
@ -517,5 +517,5 @@ No authentication required - SQLite is file-based. The file must be readable by
|
|||
| Connection URL appears in git diff | A literal credential URL was written to `ktx.yaml` | Replace it with `env:NAME` or `file:/path/to/secret` and rotate exposed credentials |
|
||||
| Database ingest returns no tables | Schema, database, or project filter is wrong, or the user lacks metadata permissions | Verify the schema list and grant metadata read permissions |
|
||||
| Query history is empty | Query history extension or warehouse history view is unavailable | Enable the warehouse-specific history feature, then rerun `ktx ingest <connectionId> --query-history` or `ktx setup` |
|
||||
| Column statistics are missing | Connector cannot access stats tables or the warehouse does not expose them | Grant stats permissions where supported; otherwise rely on schema-level context without column statistics |
|
||||
| Column statistics are missing | Connector cannot access stats tables or the warehouse does not expose them | Grant stats permissions where supported; otherwise rely on fast schema context |
|
||||
| Semantic query execution fails | Connection is missing, unreachable, or query execution is disabled | Run `ktx connection test <id>` and check the `ktx sl query` flags |
|
||||
|
|
|
|||
|
|
@ -30,36 +30,7 @@ const config = {
|
|||
};
|
||||
},
|
||||
async redirects() {
|
||||
// Alias-host canonicalization MUST come before the generic root/docs
|
||||
// redirects below. Those generic rules have no host guard, so if they ran
|
||||
// first they would inject a "/ktx" basePath into the path on the alias
|
||||
// hosts, which the alias catch-alls would then prepend a second time —
|
||||
// producing https://docs.kaelio.com/ktx/ktx/docs/... Redirects also run
|
||||
// before beforeFiles rewrites, so the ktx.sh catch-all must exclude
|
||||
// /stars* to let the stars dashboard rewrite proxy through.
|
||||
return [
|
||||
{
|
||||
source: "/slack",
|
||||
has: [{ type: "host", value: "ktx.sh" }],
|
||||
destination:
|
||||
"https://join.slack.com/t/ktxcommunity/shared_invite/zt-3y9b44m1x-LVyNNJD5nwaZHq4XS29LMQ",
|
||||
permanent: false,
|
||||
basePath: false,
|
||||
},
|
||||
{
|
||||
source: "/:path*",
|
||||
has: [{ type: "host", value: "docs.ktx.sh" }],
|
||||
destination: "https://docs.kaelio.com/ktx/:path*",
|
||||
permanent: true,
|
||||
basePath: false,
|
||||
},
|
||||
{
|
||||
source: "/:path((?!stars(?:/|$)).*)",
|
||||
has: [{ type: "host", value: "ktx.sh" }],
|
||||
destination: "https://docs.kaelio.com/ktx/:path",
|
||||
permanent: true,
|
||||
basePath: false,
|
||||
},
|
||||
{
|
||||
source: "/",
|
||||
destination: "/ktx/docs/getting-started/introduction",
|
||||
|
|
@ -72,6 +43,28 @@ const config = {
|
|||
permanent: false,
|
||||
basePath: false,
|
||||
},
|
||||
{
|
||||
source: "/:path*",
|
||||
has: [{ type: "host", value: "docs.ktx.sh" }],
|
||||
destination: "https://docs.kaelio.com/ktx/:path*",
|
||||
permanent: true,
|
||||
basePath: false,
|
||||
},
|
||||
{
|
||||
source: "/slack",
|
||||
has: [{ type: "host", value: "ktx.sh" }],
|
||||
destination:
|
||||
"https://join.slack.com/t/ktxcommunity/shared_invite/zt-3y9b44m1x-LVyNNJD5nwaZHq4XS29LMQ",
|
||||
permanent: false,
|
||||
basePath: false,
|
||||
},
|
||||
{
|
||||
source: "/:path((?!stars(?:/|$)).*)",
|
||||
has: [{ type: "host", value: "ktx.sh" }],
|
||||
destination: "https://docs.kaelio.com/ktx/:path",
|
||||
permanent: true,
|
||||
basePath: false,
|
||||
},
|
||||
];
|
||||
},
|
||||
};
|
||||
|
|
|
|||
|
|
@ -12,16 +12,15 @@
|
|||
"dependencies": {
|
||||
"@xyflow/react": "^12.10.2",
|
||||
"fumadocs-core": "16.8.10",
|
||||
"fumadocs-mdx": "15.0.7",
|
||||
"fumadocs-mdx": "15.0.4",
|
||||
"fumadocs-ui": "16.8.10",
|
||||
"html-to-image": "1.11.11",
|
||||
"next": "^16",
|
||||
"react": "19.2.6",
|
||||
"react-dom": "19.2.6"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@tailwindcss/postcss": "^4",
|
||||
"@types/node": "^25.9.1",
|
||||
"@types/node": "^25.7.0",
|
||||
"@types/react": "^19",
|
||||
"@types/react-dom": "^19",
|
||||
"tailwindcss": "^4",
|
||||
|
|
|
|||
210
docs-site/public/images/ingestion-flow-transparent.svg
Normal file
210
docs-site/public/images/ingestion-flow-transparent.svg
Normal file
|
|
@ -0,0 +1,210 @@
|
|||
<svg xmlns="http://www.w3.org/2000/svg" width="1346" height="1710" viewBox="0 0 1346 1710" role="img" aria-labelledby="title desc">
|
||||
<title id="title">ktx ingestion flow</title>
|
||||
<desc id="desc">Source systems flow through source connectors, context builder, reconciliation, and validation to create wiki Markdown and semantic-layer YAML outputs.</desc>
|
||||
<defs>
|
||||
<filter id="card-shadow" x="-12%" y="-12%" width="124%" height="124%" color-interpolation-filters="sRGB">
|
||||
<feDropShadow dx="0" dy="2" stdDeviation="2" flood-color="#0f172a" flood-opacity="0.14"/>
|
||||
</filter>
|
||||
<filter id="dark-shadow" x="-12%" y="-12%" width="124%" height="124%" color-interpolation-filters="sRGB">
|
||||
<feDropShadow dx="0" dy="2" stdDeviation="2" flood-color="#020617" flood-opacity="0.22"/>
|
||||
</filter>
|
||||
<filter id="glow-blue" x="-160%" y="-160%" width="420%" height="420%">
|
||||
<feGaussianBlur stdDeviation="7" result="blur"/>
|
||||
<feMerge>
|
||||
<feMergeNode in="blur"/>
|
||||
<feMergeNode in="SourceGraphic"/>
|
||||
</feMerge>
|
||||
</filter>
|
||||
<marker id="arrow" viewBox="0 0 10 10" refX="8.5" refY="5" markerWidth="9" markerHeight="9" orient="auto-start-reverse">
|
||||
<path d="M 0 0 L 10 5 L 0 10 z" fill="#94a3b8"/>
|
||||
</marker>
|
||||
<style>
|
||||
.card { fill: #ffffff; stroke: #e2e8f0; stroke-width: 1.4; filter: url(#card-shadow); }
|
||||
.stage { fill: #0b1f23; stroke: #17343a; stroke-width: 1.2; filter: url(#dark-shadow); }
|
||||
.title { fill: #24272d; font: 700 28px Inter, Arial, sans-serif; }
|
||||
.body { fill: #666b73; font: 500 18px Inter, Arial, sans-serif; }
|
||||
.tag { fill: #6b7280; font: 500 16px Inter, Arial, sans-serif; }
|
||||
.mono { font: 700 20px "SFMono-Regular", Consolas, monospace; }
|
||||
.stage-title { fill: #f8fafc; font: 700 28px Inter, Arial, sans-serif; }
|
||||
.stage-body { fill: #b8c6ca; font: 500 20px Inter, Arial, sans-serif; }
|
||||
.index { fill: #07313a; font: 700 22px Inter, Arial, sans-serif; text-anchor: middle; dominant-baseline: middle; }
|
||||
.edge { fill: none; stroke: #94a3b8; stroke-width: 2; stroke-linecap: round; stroke-linejoin: round; }
|
||||
.dash { fill: none; stroke: #64748b; stroke-width: 1.8; stroke-dasharray: 5 8; stroke-linecap: round; }
|
||||
</style>
|
||||
</defs>
|
||||
|
||||
<g id="source-cards">
|
||||
<g transform="translate(24 39)">
|
||||
<rect class="card" x="0" y="0" width="298" height="285" rx="4"/>
|
||||
<rect x="0" y="0" width="298" height="4" rx="2" fill="#3b82f6"/>
|
||||
<text class="title" x="22" y="52">Databases</text>
|
||||
<text class="body" x="22" y="92">Schemas, columns, keys,</text>
|
||||
<text class="body" x="22" y="120">row counts, and query</text>
|
||||
<text class="body" x="22" y="148">history.</text>
|
||||
<g transform="translate(22 180)">
|
||||
<rect x="0" y="0" width="112" height="36" rx="4" fill="#fbfaf8" stroke="#e5e1dc"/>
|
||||
<text class="tag" x="12" y="24">PostgreSQL</text>
|
||||
<rect x="120" y="0" width="100" height="36" rx="4" fill="#fbfaf8" stroke="#e5e1dc"/>
|
||||
<text class="tag" x="132" y="24">Snowflake</text>
|
||||
<rect x="0" y="46" width="92" height="36" rx="4" fill="#fbfaf8" stroke="#e5e1dc"/>
|
||||
<text class="tag" x="12" y="70">BigQuery</text>
|
||||
<rect x="100" y="46" width="74" height="36" rx="4" fill="#fbfaf8" stroke="#e5e1dc"/>
|
||||
<text class="tag" x="112" y="70">SQLite</text>
|
||||
</g>
|
||||
</g>
|
||||
|
||||
<g transform="translate(358 39)">
|
||||
<rect class="card" x="0" y="0" width="298" height="285" rx="4"/>
|
||||
<rect x="0" y="0" width="298" height="4" rx="2" fill="#f97316"/>
|
||||
<text class="title" x="22" y="52">BI tools</text>
|
||||
<text class="body" x="22" y="92">Dashboards, questions,</text>
|
||||
<text class="body" x="22" y="120">explores, usage, and trusted</text>
|
||||
<text class="body" x="22" y="148">examples.</text>
|
||||
<g transform="translate(22 180)">
|
||||
<rect x="0" y="0" width="96" height="36" rx="4" fill="#fbfaf8" stroke="#e5e1dc"/>
|
||||
<text class="tag" x="12" y="24">Metabase</text>
|
||||
<rect x="104" y="0" width="74" height="36" rx="4" fill="#fbfaf8" stroke="#e5e1dc"/>
|
||||
<text class="tag" x="116" y="24">Looker</text>
|
||||
</g>
|
||||
</g>
|
||||
|
||||
<g transform="translate(692 39)">
|
||||
<rect class="card" x="0" y="0" width="298" height="285" rx="4"/>
|
||||
<rect x="0" y="0" width="298" height="4" rx="2" fill="#f59e0b"/>
|
||||
<text class="title" x="22" y="52">Modeling code</text>
|
||||
<text class="body" x="22" y="92">Existing metrics, dimensions,</text>
|
||||
<text class="body" x="22" y="120">models, joins, and entities.</text>
|
||||
<g transform="translate(22 152)">
|
||||
<rect x="0" y="0" width="48" height="36" rx="4" fill="#fbfaf8" stroke="#e5e1dc"/>
|
||||
<text class="tag" x="12" y="24">dbt</text>
|
||||
<rect x="56" y="0" width="82" height="36" rx="4" fill="#fbfaf8" stroke="#e5e1dc"/>
|
||||
<text class="tag" x="68" y="24">LookML</text>
|
||||
<rect x="0" y="46" width="102" height="36" rx="4" fill="#fbfaf8" stroke="#e5e1dc"/>
|
||||
<text class="tag" x="12" y="70">MetricFlow</text>
|
||||
</g>
|
||||
</g>
|
||||
|
||||
<g transform="translate(1026 39)">
|
||||
<rect class="card" x="0" y="0" width="298" height="285" rx="4"/>
|
||||
<rect x="0" y="0" width="298" height="4" rx="2" fill="#10b981"/>
|
||||
<text class="title" x="22" y="52">Docs and notes</text>
|
||||
<text class="body" x="22" y="92">Policies, caveats, team</text>
|
||||
<text class="body" x="22" y="120">definitions, and analyst</text>
|
||||
<text class="body" x="22" y="148">context.</text>
|
||||
<g transform="translate(22 180)">
|
||||
<rect x="0" y="0" width="72" height="36" rx="4" fill="#fbfaf8" stroke="#e5e1dc"/>
|
||||
<text class="tag" x="12" y="24">Notion</text>
|
||||
<rect x="80" y="0" width="84" height="36" rx="4" fill="#fbfaf8" stroke="#e5e1dc"/>
|
||||
<text class="tag" x="92" y="24">Any text</text>
|
||||
</g>
|
||||
</g>
|
||||
</g>
|
||||
|
||||
<g id="edges">
|
||||
<path class="edge" d="M172 324 V380 Q172 394 186 394 H507 Q507 394 507 380 V324"/>
|
||||
<path class="edge" d="M841 324 V380 Q841 394 827 394 H507"/>
|
||||
<path class="edge" d="M1175 324 V380 Q1175 394 1161 394 H673 Q673 394 673 408 V433" marker-end="url(#arrow)"/>
|
||||
<path class="edge" d="M507 394 H673"/>
|
||||
<path class="edge" d="M673 618 V651" marker-end="url(#arrow)"/>
|
||||
<path class="edge" d="M673 833 V866" marker-end="url(#arrow)"/>
|
||||
<path class="edge" d="M673 1048 V1081" marker-end="url(#arrow)"/>
|
||||
<path class="edge" d="M673 1262 V1310 Q673 1325 656 1325 H305 Q291 1325 291 1339 V1364" marker-end="url(#arrow)"/>
|
||||
<path class="edge" d="M673 1262 V1310 Q673 1325 690 1325 H1043 Q1057 1325 1057 1339 V1364" marker-end="url(#arrow)"/>
|
||||
<path class="dash" d="M546 1523 H800"/>
|
||||
<path d="M546 1523 l9 -6 v12 z" fill="#64748b"/>
|
||||
<path d="M800 1523 l-9 -6 v12 z" fill="#64748b"/>
|
||||
</g>
|
||||
|
||||
<g id="particles">
|
||||
<circle cx="256" cy="394" r="18" fill="#3b82f6" opacity="0.18" filter="url(#glow-blue)"/>
|
||||
<circle cx="256" cy="394" r="6" fill="#3b82f6" opacity="0.9"/>
|
||||
<circle cx="632" cy="394" r="18" fill="#f97316" opacity="0.18" filter="url(#glow-blue)"/>
|
||||
<circle cx="632" cy="394" r="6" fill="#f97316" opacity="0.9"/>
|
||||
<circle cx="830" cy="394" r="18" fill="#10b981" opacity="0.18" filter="url(#glow-blue)"/>
|
||||
<circle cx="830" cy="394" r="6" fill="#10b981" opacity="0.9"/>
|
||||
<circle cx="673" cy="635" r="17" fill="#10b981" opacity="0.18" filter="url(#glow-blue)"/>
|
||||
<circle cx="673" cy="635" r="6" fill="#10b981" opacity="0.9"/>
|
||||
<circle cx="673" cy="1065" r="17" fill="#f59e0b" opacity="0.18" filter="url(#glow-blue)"/>
|
||||
<circle cx="673" cy="1065" r="6" fill="#f59e0b" opacity="0.9"/>
|
||||
<circle cx="573" cy="1322" r="17" fill="#3b82f6" opacity="0.18" filter="url(#glow-blue)"/>
|
||||
<circle cx="573" cy="1322" r="6" fill="#3b82f6" opacity="0.9"/>
|
||||
</g>
|
||||
|
||||
<g id="stages">
|
||||
<g transform="translate(464 438)">
|
||||
<rect class="stage" x="0" y="0" width="420" height="180" rx="4"/>
|
||||
<circle cx="52" cy="90" r="26" fill="#55dced"/>
|
||||
<text class="index" x="52" y="90">1</text>
|
||||
<text class="stage-title" x="98" y="72">Source connectors</text>
|
||||
<text class="stage-body" x="98" y="110">Read each configured system in</text>
|
||||
<text class="stage-body" x="98" y="140">its native shape.</text>
|
||||
</g>
|
||||
|
||||
<g transform="translate(464 653)">
|
||||
<rect class="stage" x="0" y="0" width="420" height="180" rx="4"/>
|
||||
<circle cx="52" cy="90" r="26" fill="#55dced"/>
|
||||
<text class="index" x="52" y="90">2</text>
|
||||
<text class="stage-title" x="98" y="72">Context builder</text>
|
||||
<text class="stage-body" x="98" y="110">Turn source evidence into</text>
|
||||
<text class="stage-body" x="98" y="140">proposed context updates.</text>
|
||||
</g>
|
||||
|
||||
<g transform="translate(464 868)">
|
||||
<rect class="stage" x="0" y="0" width="420" height="180" rx="4"/>
|
||||
<circle cx="52" cy="90" r="26" fill="#55dced"/>
|
||||
<text class="index" x="52" y="90">3</text>
|
||||
<text class="stage-title" x="98" y="72">Reconciliation</text>
|
||||
<text class="stage-body" x="98" y="110">Merge new evidence with the</text>
|
||||
<text class="stage-body" x="98" y="140">context that already exists.</text>
|
||||
</g>
|
||||
|
||||
<g transform="translate(464 1082)">
|
||||
<rect class="stage" x="0" y="0" width="420" height="180" rx="4"/>
|
||||
<circle cx="52" cy="90" r="26" fill="#55dced"/>
|
||||
<text class="index" x="52" y="90">4</text>
|
||||
<text class="stage-title" x="98" y="72">Validation</text>
|
||||
<text class="stage-body" x="98" y="110">Check references and semantics</text>
|
||||
<text class="stage-body" x="98" y="140">before agents rely on them.</text>
|
||||
</g>
|
||||
</g>
|
||||
|
||||
<g id="outputs">
|
||||
<g transform="translate(60 1373)">
|
||||
<rect class="card" x="0" y="0" width="485" height="329" rx="4"/>
|
||||
<rect x="0" y="0" width="485" height="4" rx="2" fill="#10b981"/>
|
||||
<text class="mono" x="24" y="52" fill="#10b981">wiki/*.md</text>
|
||||
<text class="title" x="24" y="100">Wiki</text>
|
||||
<g transform="translate(24 122)">
|
||||
<rect x="0" y="0" width="90" height="36" rx="4" fill="#fbfaf8" stroke="#e5e1dc"/>
|
||||
<text class="tag" x="12" y="24">free-form</text>
|
||||
<rect x="98" y="0" width="140" height="36" rx="4" fill="#fbfaf8" stroke="#e5e1dc"/>
|
||||
<text class="tag" x="110" y="24">auto-maintained</text>
|
||||
</g>
|
||||
<text class="body" x="24" y="194">Definitions, caveats, policies, analyst notes, and</text>
|
||||
<text class="body" x="24" y="222">business language that agents can search.</text>
|
||||
</g>
|
||||
|
||||
<g transform="translate(803 1373)">
|
||||
<rect class="card" x="0" y="0" width="485" height="329" rx="4"/>
|
||||
<rect x="0" y="0" width="485" height="4" rx="2" fill="#3b82f6"/>
|
||||
<text class="mono" x="24" y="52" fill="#3b82f6">semantic-layer/*.yaml</text>
|
||||
<text class="title" x="24" y="100">Semantic layer</text>
|
||||
<g transform="translate(24 122)">
|
||||
<rect x="0" y="0" width="96" height="36" rx="4" fill="#fbfaf8" stroke="#e5e1dc"/>
|
||||
<text class="tag" x="12" y="24">structured</text>
|
||||
<rect x="104" y="0" width="104" height="36" rx="4" fill="#fbfaf8" stroke="#e5e1dc"/>
|
||||
<text class="tag" x="116" y="24">executable</text>
|
||||
<rect x="216" y="0" width="140" height="36" rx="4" fill="#fbfaf8" stroke="#e5e1dc"/>
|
||||
<text class="tag" x="228" y="24">auto-maintained</text>
|
||||
</g>
|
||||
<text class="body" x="24" y="194">Metrics, joins, tables, dimensions, filters, and</text>
|
||||
<text class="body" x="24" y="222">segments that ktx can validate and compile into</text>
|
||||
<text class="body" x="24" y="250">SQL.</text>
|
||||
</g>
|
||||
|
||||
<g transform="translate(622 1505)">
|
||||
<rect x="0" y="0" width="102" height="36" rx="4" fill="#ffffff" stroke="#e5e1dc"/>
|
||||
<text class="tag" x="13" y="24">references</text>
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 12 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 346 KiB After Width: | Height: | Size: 137 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 176 KiB |
|
|
@ -2,8 +2,6 @@ import assert from "node:assert/strict";
|
|||
import { spawn } from "node:child_process";
|
||||
import { once } from "node:events";
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import http from "node:http";
|
||||
import https from "node:https";
|
||||
import { dirname, join } from "node:path";
|
||||
import { createServer } from "node:net";
|
||||
import { after, before, test } from "node:test";
|
||||
|
|
@ -102,37 +100,6 @@ after(async () => {
|
|||
}
|
||||
});
|
||||
|
||||
// Node's fetch (undici) overwrites the Host header with the connection host,
|
||||
// so the alias-host redirect rules never match. The low-level http(s) client
|
||||
// sends Host verbatim, which is what the alias canonicalization keys off of.
|
||||
function requestWithHost(hostHeader, path) {
|
||||
const target = new URL(docsSiteUrl);
|
||||
const client = target.protocol === "https:" ? https : http;
|
||||
const port =
|
||||
target.port || (target.protocol === "https:" ? "443" : "80");
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const request = client.request(
|
||||
{
|
||||
hostname: target.hostname,
|
||||
port,
|
||||
path,
|
||||
method: "GET",
|
||||
headers: { Host: hostHeader },
|
||||
},
|
||||
(response) => {
|
||||
response.resume();
|
||||
resolve({
|
||||
status: response.statusCode,
|
||||
location: response.headers.location,
|
||||
});
|
||||
},
|
||||
);
|
||||
request.on("error", reject);
|
||||
request.end();
|
||||
});
|
||||
}
|
||||
|
||||
test("/ktx/docs redirects to the docs introduction", async () => {
|
||||
const response = await fetch(`${docsSiteUrl}${docsBasePath}/docs`, {
|
||||
redirect: "manual",
|
||||
|
|
@ -174,51 +141,3 @@ test("/ktx/api/search returns docs search results", async () => {
|
|||
"search should return at least one docs result",
|
||||
);
|
||||
});
|
||||
|
||||
test("ktx.sh canonicalizes to a single /ktx basePath on the docs host", async () => {
|
||||
const root = await requestWithHost("ktx.sh", "/");
|
||||
assert.equal(root.status, 308);
|
||||
assert.equal(root.location, "https://docs.kaelio.com/ktx/");
|
||||
assert.ok(
|
||||
!root.location.includes("/ktx/ktx"),
|
||||
"the basePath must not be doubled",
|
||||
);
|
||||
|
||||
const page = await requestWithHost(
|
||||
"ktx.sh",
|
||||
"/docs/getting-started/quickstart",
|
||||
);
|
||||
assert.equal(page.status, 308);
|
||||
assert.equal(
|
||||
page.location,
|
||||
"https://docs.kaelio.com/ktx/docs/getting-started/quickstart",
|
||||
);
|
||||
});
|
||||
|
||||
test("docs.ktx.sh canonicalizes to a single /ktx basePath on the docs host", async () => {
|
||||
const root = await requestWithHost("docs.ktx.sh", "/");
|
||||
assert.equal(root.status, 308);
|
||||
assert.equal(root.location, "https://docs.kaelio.com/ktx");
|
||||
assert.ok(
|
||||
!root.location.includes("/ktx/ktx"),
|
||||
"the basePath must not be doubled",
|
||||
);
|
||||
|
||||
const page = await requestWithHost("docs.ktx.sh", "/llms.txt");
|
||||
assert.equal(page.status, 308);
|
||||
assert.equal(page.location, "https://docs.kaelio.com/ktx/llms.txt");
|
||||
});
|
||||
|
||||
test("ktx.sh keeps the /slack and /stars exceptions", async () => {
|
||||
const slack = await requestWithHost("ktx.sh", "/slack");
|
||||
assert.equal(slack.status, 307);
|
||||
assert.match(slack.location, /^https:\/\/join\.slack\.com\//);
|
||||
|
||||
// /stars is proxied by a beforeFiles rewrite, so the apex catch-all must not
|
||||
// canonicalize it to the docs host.
|
||||
const stars = await requestWithHost("ktx.sh", "/stars");
|
||||
assert.ok(
|
||||
!(stars.location ?? "").startsWith("https://docs.kaelio.com"),
|
||||
"the stars dashboard must not be redirected to the docs host",
|
||||
);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -85,7 +85,7 @@ test("product mechanics component explains ingestion outputs", async () => {
|
|||
"compile into SQL",
|
||||
'"use client"',
|
||||
"@xyflow/react",
|
||||
"<FlowCanvas",
|
||||
"<ReactFlow",
|
||||
"getSmoothStepPath",
|
||||
"animateMotion",
|
||||
"mechanics-particle",
|
||||
|
|
@ -97,21 +97,21 @@ test("product mechanics component explains ingestion outputs", async () => {
|
|||
);
|
||||
}
|
||||
|
||||
// The ReactFlow canvas config lives in the shared FlowCanvas wrapper, which
|
||||
// product-mechanics renders. Assert the static read-only behavior there.
|
||||
const flowCanvas = await readDocsFile("components/flow-canvas.tsx");
|
||||
for (const guard of [
|
||||
assert.match(
|
||||
component,
|
||||
/nodesDraggable=\{false\}/,
|
||||
/nodesConnectable=\{false\}/,
|
||||
"ReactFlow canvas should disable node dragging",
|
||||
);
|
||||
assert.match(
|
||||
component,
|
||||
/panOnDrag=\{false\}/,
|
||||
"ReactFlow canvas should disable panning",
|
||||
);
|
||||
assert.match(
|
||||
component,
|
||||
/zoomOnScroll=\{false\}/,
|
||||
/elementsSelectable=\{false\}/,
|
||||
]) {
|
||||
assert.match(
|
||||
flowCanvas,
|
||||
guard,
|
||||
`shared FlowCanvas should enforce static read-only behavior: ${guard}`,
|
||||
);
|
||||
}
|
||||
"ReactFlow canvas should disable scroll zoom",
|
||||
);
|
||||
|
||||
assert.doesNotMatch(component, /raw-sources/);
|
||||
assert.doesNotMatch(component, /\.ktx/);
|
||||
|
|
|
|||
|
|
@ -1,74 +0,0 @@
|
|||
import assert from "node:assert/strict";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { dirname, join } from "node:path";
|
||||
import { test } from "node:test";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const docsSiteDir = join(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
|
||||
async function readDocsFile(path) {
|
||||
return readFile(join(docsSiteDir, path), "utf8");
|
||||
}
|
||||
|
||||
test("docs introduction renders the serving phase after ingestion", async () => {
|
||||
const introduction = await readDocsFile(
|
||||
"content/docs/getting-started/introduction.mdx",
|
||||
);
|
||||
|
||||
assert.match(
|
||||
introduction,
|
||||
/import\s+\{\s*ProductRuntime\s*\}\s+from\s+"@\/components\/product-runtime";/,
|
||||
);
|
||||
assert.match(introduction, /<ProductRuntime\s*\/>/);
|
||||
|
||||
const mechanicsIndex = introduction.indexOf("<ProductMechanics />");
|
||||
const runtimeIndex = introduction.indexOf("<ProductRuntime />");
|
||||
const useCaseIndex = introduction.indexOf("## Use it for");
|
||||
|
||||
assert.ok(
|
||||
runtimeIndex > mechanicsIndex,
|
||||
"serving diagram should appear after the ingestion diagram",
|
||||
);
|
||||
assert.ok(
|
||||
runtimeIndex < useCaseIndex,
|
||||
"serving diagram should appear before use-case sections",
|
||||
);
|
||||
});
|
||||
|
||||
test("product runtime component explains the serving cycle", async () => {
|
||||
const component = await readDocsFile("components/product-runtime.tsx");
|
||||
|
||||
for (const expectedText of [
|
||||
"How serving works",
|
||||
"Serving flow",
|
||||
"From an agent request to a governed answer",
|
||||
"Your agent",
|
||||
"Claude Code",
|
||||
"Cursor",
|
||||
"Codex",
|
||||
"Search wiki + semantic layer",
|
||||
"Return approved metrics",
|
||||
"Compile metrics → SQL",
|
||||
"Context layer",
|
||||
"Database",
|
||||
"search + read",
|
||||
"read-only",
|
||||
"wiki/*.md",
|
||||
"semantic-layer/*.yaml",
|
||||
'"use client"',
|
||||
"@xyflow/react",
|
||||
"FlowCanvas",
|
||||
"getSmoothStepPath",
|
||||
"animateMotion",
|
||||
"runtime-particle",
|
||||
"buildCyclePath",
|
||||
]) {
|
||||
assert.ok(
|
||||
component.includes(expectedText),
|
||||
`component should include: ${expectedText}`,
|
||||
);
|
||||
}
|
||||
|
||||
assert.doesNotMatch(component, /raw-sources/);
|
||||
assert.doesNotMatch(component, /<img/);
|
||||
});
|
||||
|
|
@ -77,6 +77,8 @@ maintains, validates, and serves that layer.
|
|||
| Connection ref in prose | **connection id** (lowercase, two words) | "connection ID" |
|
||||
| CLI arg/flag literal | `connectionId` (code font) | — |
|
||||
| File path placeholder | `<connection-id>` (code font) | — |
|
||||
| Fast schema mode | **fast ingest** | schema ingest, schema-only ingest |
|
||||
| AI-enriched mode | **deep ingest** | AI-enriched ingest |
|
||||
| Ingest of a primary connection | **database ingest** | — |
|
||||
| Ingest of a context-source connection | **context-source ingest** | bare "source ingest" |
|
||||
| Wiki capture | **text ingest** | — |
|
||||
|
|
|
|||
|
|
@ -37,9 +37,6 @@
|
|||
"@semantic-release/release-notes-generator",
|
||||
"conventional-changelog-conventionalcommits"
|
||||
],
|
||||
"ignore": [
|
||||
".context/**"
|
||||
],
|
||||
"ignoreBinaries": [
|
||||
"uv",
|
||||
"lsof"
|
||||
|
|
|
|||
12
package.json
12
package.json
|
|
@ -1,10 +1,10 @@
|
|||
{
|
||||
"name": "ktx-workspace",
|
||||
"version": "0.9.0",
|
||||
"version": "0.7.0",
|
||||
"description": "Workspace root for ktx packages",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"packageManager": "pnpm@11.4.0",
|
||||
"packageManager": "pnpm@11.1.1",
|
||||
"engines": {
|
||||
"node": ">=22.0.0",
|
||||
"pnpm": ">=10.20.0"
|
||||
|
|
@ -24,7 +24,6 @@
|
|||
"dead-code:fix": "biome check . --formatter-enabled=false --assist-enabled=false --write && knip --fix --format",
|
||||
"dead-code:knip": "knip --reporter compact",
|
||||
"dead-code:knip:production": "knip --production --reporter compact",
|
||||
"deps:upgrade": "node scripts/upgrade-dependencies.mjs",
|
||||
"docs": "kill $(lsof -ti:3000) 2>/dev/null; pnpm --filter ktx-docs run dev",
|
||||
"ktx": "node scripts/run-ktx.mjs",
|
||||
"link:dev": "node scripts/link-dev-cli.mjs",
|
||||
|
|
@ -32,7 +31,6 @@
|
|||
"setup:dev": "node scripts/setup-dev.mjs",
|
||||
"release:published-smoke": "node scripts/published-package-smoke.mjs --require-config",
|
||||
"release:local-embeddings-smoke": "node scripts/local-embeddings-runtime-smoke.mjs --require-opt-in",
|
||||
"release:codex-backend-smoke": "node scripts/codex-backend-live-smoke.mjs",
|
||||
"release:readiness": "node scripts/release-readiness.mjs",
|
||||
"release:update-version": "node scripts/update-public-release-version.mjs",
|
||||
"relationships:acquire-public-fixtures": "node scripts/acquire-public-benchmark-fixtures.mjs",
|
||||
|
|
@ -60,11 +58,11 @@
|
|||
"@semantic-release/github": "^12.0.8",
|
||||
"@semantic-release/npm": "^13.1.5",
|
||||
"@semantic-release/release-notes-generator": "^14.1.1",
|
||||
"@types/node": "^25.9.1",
|
||||
"@types/node": "^25.7.0",
|
||||
"better-sqlite3": "^12.10.0",
|
||||
"conventional-changelog-conventionalcommits": "^9.3.1",
|
||||
"knip": "^6.14.1",
|
||||
"pg": "^8.21.0",
|
||||
"knip": "^6.12.2",
|
||||
"pg": "^8.20.0",
|
||||
"semantic-release": "^25.0.3",
|
||||
"typescript": "^6.0.3",
|
||||
"yaml": "^2.9.0"
|
||||
|
|
|
|||
|
|
@ -1,11 +1,7 @@
|
|||
{
|
||||
"name": "@kaelio/ktx",
|
||||
"version": "0.9.0",
|
||||
"version": "0.7.0",
|
||||
"description": "Standalone ktx context layer for data agents",
|
||||
"author": {
|
||||
"name": "Kaelio",
|
||||
"url": "https://www.kaelio.com"
|
||||
},
|
||||
"type": "module",
|
||||
"engines": {
|
||||
"node": ">=22.0.0"
|
||||
|
|
@ -47,39 +43,36 @@
|
|||
"search:pglite-sl-prototype": "node ../../scripts/pglite-sl-search-prototype.mjs"
|
||||
},
|
||||
"dependencies": {
|
||||
"@ai-sdk/anthropic": "3.0.78",
|
||||
"@ai-sdk/devtools": "0.0.18",
|
||||
"@ai-sdk/google-vertex": "^4.0.134",
|
||||
"@anthropic-ai/claude-agent-sdk": "0.3.146",
|
||||
"@clack/core": "1.3.1",
|
||||
"@ai-sdk/anthropic": "3.0.77",
|
||||
"@ai-sdk/devtools": "0.0.17",
|
||||
"@ai-sdk/google-vertex": "^4.0.128",
|
||||
"@anthropic-ai/claude-agent-sdk": "0.3.142",
|
||||
"@clack/prompts": "1.4.0",
|
||||
"@clickhouse/client": "^1.18.5",
|
||||
"@clickhouse/client": "^1.18.4",
|
||||
"@commander-js/extra-typings": "14.0.0",
|
||||
"@google-cloud/bigquery": "^8.3.1",
|
||||
"@looker/sdk": "^26.8.0",
|
||||
"@looker/sdk-node": "^26.8.0",
|
||||
"@looker/sdk-rtl": "^21.6.5",
|
||||
"@modelcontextprotocol/sdk": "^1.29.0",
|
||||
"@notionhq/client": "^5.22.0",
|
||||
"@openai/codex-sdk": "^0.133.0",
|
||||
"ai": "^6.0.188",
|
||||
"@notionhq/client": "^5.21.0",
|
||||
"ai": "^6.0.180",
|
||||
"better-sqlite3": "^12.10.0",
|
||||
"commander": "14.0.3",
|
||||
"fflate": "^0.8.3",
|
||||
"fflate": "^0.8.2",
|
||||
"handlebars": "^4.7.9",
|
||||
"ink": "^7.0.3",
|
||||
"ink": "^7.0.2",
|
||||
"lookml-parser": "7.1.0",
|
||||
"minimatch": "^10.2.5",
|
||||
"mssql": "^12.5.4",
|
||||
"mssql": "^12.5.2",
|
||||
"mysql2": "^3.22.3",
|
||||
"openai": "^6.38.0",
|
||||
"openai": "^6.37.0",
|
||||
"p-limit": "^7.3.0",
|
||||
"pg": "^8.21.0",
|
||||
"posthog-node": "^5.34.9",
|
||||
"pg": "^8.20.0",
|
||||
"posthog-node": "^5.0.0",
|
||||
"react": "^19.2.6",
|
||||
"semver": "^7.8.1",
|
||||
"simple-git": "3.36.0",
|
||||
"snowflake-sdk": "^2.4.2",
|
||||
"snowflake-sdk": "^2.4.1",
|
||||
"yaml": "^2.9.0",
|
||||
"zod": "^4.4.3"
|
||||
},
|
||||
|
|
@ -88,15 +81,14 @@
|
|||
"@electric-sql/pglite-socket": "^0.1.5",
|
||||
"@types/better-sqlite3": "^7.6.13",
|
||||
"@types/mssql": "^12.3.0",
|
||||
"@types/node": "^25.9.1",
|
||||
"@types/node": "^25.7.0",
|
||||
"@types/pg": "^8.20.0",
|
||||
"@types/react": "^19.2.15",
|
||||
"@types/semver": "^7.7.1",
|
||||
"@vitest/coverage-v8": "^4.1.7",
|
||||
"@types/react": "^19.2.14",
|
||||
"@vitest/coverage-v8": "^4.1.6",
|
||||
"ajv": "8.20.0",
|
||||
"ink-testing-library": "^4.0.0",
|
||||
"typescript": "^6.0.3",
|
||||
"vitest": "^4.1.7"
|
||||
"vitest": "^4.1.6"
|
||||
},
|
||||
"license": "Apache-2.0",
|
||||
"repository": {
|
||||
|
|
|
|||
|
|
@ -3,30 +3,6 @@ import type { KtxCliIo } from './cli-runtime.js';
|
|||
|
||||
const ESC = String.fromCharCode(0x1b);
|
||||
|
||||
export interface CliStyleEnv {
|
||||
NO_COLOR?: string;
|
||||
TERM?: string;
|
||||
}
|
||||
|
||||
function ansiEnabled(env: CliStyleEnv = process.env): boolean {
|
||||
return !env.NO_COLOR && env.TERM !== 'dumb';
|
||||
}
|
||||
|
||||
function ansiColor(text: string, open: number, close: number, env?: CliStyleEnv): string {
|
||||
if (!ansiEnabled(env)) {
|
||||
return text;
|
||||
}
|
||||
return `${ESC}[${open}m${text}${ESC}[${close}m`;
|
||||
}
|
||||
|
||||
export function dim(text: string, env?: CliStyleEnv): string {
|
||||
return ansiColor(text, 2, 22, env);
|
||||
}
|
||||
|
||||
export function cyan(text: string, env?: CliStyleEnv): string {
|
||||
return ansiColor(text, 36, 39, env);
|
||||
}
|
||||
|
||||
export interface RailBufferedSource {
|
||||
stdoutText(): string;
|
||||
stderrText(): string;
|
||||
|
|
@ -85,11 +61,11 @@ export function createClackSpinner(): KtxCliSpinner {
|
|||
}
|
||||
|
||||
function magenta(text: string): string {
|
||||
return ansiColor(text, 35, 39);
|
||||
return `${ESC}[35m${text}${ESC}[39m`;
|
||||
}
|
||||
|
||||
function red(text: string): string {
|
||||
return ansiColor(text, 31, 39);
|
||||
return `${ESC}[31m${text}${ESC}[39m`;
|
||||
}
|
||||
|
||||
export function createStaticCliSpinner(io: KtxCliSpinnerIo): KtxCliSpinner {
|
||||
|
|
|
|||
|
|
@ -2,7 +2,6 @@ import { existsSync } from 'node:fs';
|
|||
import { join } from 'node:path';
|
||||
import { Command, type CommandUnknownOpts, InvalidArgumentError } from '@commander-js/extra-typings';
|
||||
import type { KtxCliDeps, KtxCliIo, KtxCliPackageInfo } from './cli-runtime.js';
|
||||
import { registerCompletionCommands } from './commands/completion-commands.js';
|
||||
import { registerConnectionCommands } from './commands/connection-commands.js';
|
||||
import { registerIngestCommands } from './commands/ingest-commands.js';
|
||||
import { registerWikiCommands } from './commands/knowledge-commands.js';
|
||||
|
|
@ -16,7 +15,6 @@ import { renderMissingProjectMessage } from './doctor.js';
|
|||
import { findNearestKtxProjectDir, resolveKtxProjectDir } from './project-resolver.js';
|
||||
import { profileMark, profileSpan } from './startup-profile.js';
|
||||
import type { CommandOutcome } from './telemetry/index.js';
|
||||
import { prepareUpdateCheckNotice, type PrepareUpdateCheckNoticeOptions } from './update-check/update-check.js';
|
||||
|
||||
profileMark('module:cli-program');
|
||||
|
||||
|
|
@ -40,8 +38,6 @@ interface KtxCommanderProgramOptions {
|
|||
runInit: (args: { projectDir: string; force: boolean }, io: KtxCliIo) => Promise<number>;
|
||||
}
|
||||
|
||||
type KtxCliUpdateCheckOptions = Pick<PrepareUpdateCheckNoticeOptions, 'env' | 'fetchDistTags' | 'homeDir' | 'now'>;
|
||||
|
||||
export interface BuildKtxProgramOptions {
|
||||
io: KtxCliIo;
|
||||
deps: KtxCliDeps;
|
||||
|
|
@ -50,7 +46,6 @@ export interface BuildKtxProgramOptions {
|
|||
setExitCode?: (code: number) => void;
|
||||
argv?: string[];
|
||||
setTelemetryModule?: (telemetry: typeof import('./telemetry/index.js')) => void;
|
||||
updateCheck?: KtxCliUpdateCheckOptions;
|
||||
}
|
||||
|
||||
type CommanderExitLike = { exitCode: number; code: string; message: string };
|
||||
|
|
@ -435,29 +430,11 @@ export function collectCommandFlagsPresent(command: CommandUnknownOpts): Record<
|
|||
|
||||
export function buildKtxProgram(options: BuildKtxProgramOptions): Command {
|
||||
const program = createBaseProgram(options.packageInfo, options.io);
|
||||
let pendingUpdateNotice: string | null = null;
|
||||
|
||||
program.hook('preAction', async (_thisCommand, actionCommand) => {
|
||||
// The hidden completion command must stay silent and side-effect free: skip
|
||||
// the telemetry notice, command span, project checks, and update checks entirely.
|
||||
if (commandPath(actionCommand as CommandPathNode).includes('__complete')) {
|
||||
return;
|
||||
}
|
||||
const commandNode = actionCommand as CommandPathNode;
|
||||
const updateCheck = await prepareUpdateCheckNotice({
|
||||
io: options.io,
|
||||
env: options.updateCheck?.env,
|
||||
fetchDistTags: options.updateCheck?.fetchDistTags,
|
||||
homeDir: options.updateCheck?.homeDir,
|
||||
installedVersion: options.packageInfo.version,
|
||||
now: options.updateCheck?.now,
|
||||
commandOptions: commandOptions(commandNode),
|
||||
});
|
||||
pendingUpdateNotice = updateCheck.notice;
|
||||
|
||||
const telemetry = await import('./telemetry/index.js');
|
||||
options.setTelemetryModule?.(telemetry);
|
||||
await telemetry.showTelemetryNoticeIfNeeded(options.io, options.packageInfo);
|
||||
const commandNode = actionCommand as CommandPathNode;
|
||||
const path = commandPath(commandNode);
|
||||
const projectDir = resolveCommandProjectDir(commandNode);
|
||||
const hasProject = ktxYamlExists(projectDir);
|
||||
|
|
@ -474,13 +451,6 @@ export function buildKtxProgram(options: BuildKtxProgramOptions): Command {
|
|||
ensureProjectAvailable(options.io, commandNode);
|
||||
});
|
||||
|
||||
program.hook('postAction', () => {
|
||||
if (pendingUpdateNotice) {
|
||||
options.io.stderr.write(pendingUpdateNotice);
|
||||
pendingUpdateNotice = null;
|
||||
}
|
||||
});
|
||||
|
||||
const context: KtxCliCommandContext = {
|
||||
io: options.io,
|
||||
deps: options.deps,
|
||||
|
|
@ -506,7 +476,6 @@ export function buildKtxProgram(options: BuildKtxProgramOptions): Command {
|
|||
registerStatusCommands(program, context);
|
||||
registerMcpCommands(program, context);
|
||||
registerAdminCommands(program, context);
|
||||
registerCompletionCommands(program, context);
|
||||
|
||||
return program;
|
||||
}
|
||||
|
|
@ -553,13 +522,6 @@ export async function runCommanderKtxCli(
|
|||
try {
|
||||
return await runBareInteractiveCommand(program, io, context);
|
||||
} catch (error) {
|
||||
const telemetry = await import('./telemetry/index.js');
|
||||
await telemetry.reportException({
|
||||
error,
|
||||
context: { source: 'bare-interactive', handled: true, fatal: false },
|
||||
packageInfo: info,
|
||||
io,
|
||||
});
|
||||
io.stderr.write(`${formatCliError(error)}\n`);
|
||||
return 1;
|
||||
}
|
||||
|
|
@ -594,23 +556,6 @@ export async function runCommanderKtxCli(
|
|||
outcome: commandOutcomeForParseResult(parseError, exitCode),
|
||||
error: parseError,
|
||||
});
|
||||
if (
|
||||
parseError &&
|
||||
!isCommanderExit(parseError) &&
|
||||
!isKtxProjectMissingAbortError(parseError)
|
||||
) {
|
||||
await telemetryModule.reportException({
|
||||
error: parseError,
|
||||
context: {
|
||||
source: completed?.commandPath.join(' ') ?? 'commander parseAsync',
|
||||
handled: true,
|
||||
fatal: false,
|
||||
},
|
||||
projectDir: completed?.projectGroupAttached ? completed.projectDir : undefined,
|
||||
packageInfo: info,
|
||||
io,
|
||||
});
|
||||
}
|
||||
await telemetryModule.emitCompletedCommand({ completed, packageInfo: info, io });
|
||||
await telemetryModule.shutdownTelemetryEmitter();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -89,88 +89,6 @@ export async function runInitForCommander(
|
|||
return await runInit(args, io);
|
||||
}
|
||||
|
||||
function signalExitCode(signal: NodeJS.Signals): number {
|
||||
// 128 + signal number: SIGINT (2) -> 130, SIGTERM (15) -> 143.
|
||||
return signal === 'SIGTERM' ? 143 : 130;
|
||||
}
|
||||
|
||||
/**
|
||||
* Flush telemetry on interrupt for the real CLI process. `capture()` is
|
||||
* fire-and-forget and the only flush guarantee lives in a `finally` a signal
|
||||
* skips, so Ctrl-C / `kill` of a long-running command (ingest, `mcp stdio`)
|
||||
* would otherwise drop its `command` event and queued events. Installed only
|
||||
* when driving the actual process; programmatic/test callers pass their own
|
||||
* `io` and never reach here. Returns a disposer that removes the listeners.
|
||||
*/
|
||||
function installTelemetrySignalFlush(io: KtxCliIo, info: KtxCliPackageInfo): () => void {
|
||||
let handling = false;
|
||||
const handle = (signal: NodeJS.Signals): void => {
|
||||
if (handling) {
|
||||
process.exit(signalExitCode(signal));
|
||||
}
|
||||
handling = true;
|
||||
void (async () => {
|
||||
try {
|
||||
const { emitAbortedCommandAndShutdown } = await import('./telemetry/index.js');
|
||||
await emitAbortedCommandAndShutdown({ packageInfo: info, io });
|
||||
} catch {
|
||||
// Best-effort: never let a telemetry hiccup block the interrupt exit.
|
||||
}
|
||||
process.exit(signalExitCode(signal));
|
||||
})();
|
||||
};
|
||||
const onSigint = (): void => handle('SIGINT');
|
||||
const onSigterm = (): void => handle('SIGTERM');
|
||||
process.on('SIGINT', onSigint);
|
||||
process.on('SIGTERM', onSigterm);
|
||||
return () => {
|
||||
process.off('SIGINT', onSigint);
|
||||
process.off('SIGTERM', onSigterm);
|
||||
};
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function createGlobalExceptionReporter(io: KtxCliIo, info: KtxCliPackageInfo) {
|
||||
return async (source: 'uncaughtException' | 'unhandledRejection', error: unknown): Promise<void> => {
|
||||
const { reportException, shutdownTelemetryEmitter } = await import('./telemetry/index.js');
|
||||
await reportException({
|
||||
error,
|
||||
context: { source, handled: false, fatal: true },
|
||||
io,
|
||||
packageInfo: info,
|
||||
immediate: true,
|
||||
});
|
||||
await shutdownTelemetryEmitter();
|
||||
};
|
||||
}
|
||||
|
||||
export function installGlobalExceptionHandlers(io: KtxCliIo, info: KtxCliPackageInfo): () => void {
|
||||
const report = createGlobalExceptionReporter(io, info);
|
||||
const handle = (source: 'uncaughtException' | 'unhandledRejection', error: unknown): void => {
|
||||
void (async () => {
|
||||
try {
|
||||
await report(source, error);
|
||||
} catch {
|
||||
// Best-effort: preserve Node's process termination behavior.
|
||||
}
|
||||
if (error instanceof Error && error.stack) {
|
||||
io.stderr.write(`${error.stack}\n`);
|
||||
} else {
|
||||
io.stderr.write(`${String(error)}\n`);
|
||||
}
|
||||
process.exit(1);
|
||||
})();
|
||||
};
|
||||
const onUncaught = (error: Error): void => handle('uncaughtException', error);
|
||||
const onUnhandled = (reason: unknown): void => handle('unhandledRejection', reason);
|
||||
process.on('uncaughtException', onUncaught);
|
||||
process.on('unhandledRejection', onUnhandled);
|
||||
return () => {
|
||||
process.off('uncaughtException', onUncaught);
|
||||
process.off('unhandledRejection', onUnhandled);
|
||||
};
|
||||
}
|
||||
|
||||
export async function runKtxCli(
|
||||
argv = process.argv.slice(2),
|
||||
io: KtxCliIo = process,
|
||||
|
|
@ -180,17 +98,7 @@ export async function runKtxCli(
|
|||
profileMark('runtime:runKtxCli');
|
||||
const { runCommanderKtxCli } = await profileSpan('import ./cli-program.js', () => import('./cli-program.js'));
|
||||
|
||||
// Real-process entry only: flush telemetry if interrupted. Test/programmatic
|
||||
// callers pass their own `io`, so they never install process-level handlers.
|
||||
const removeSignalFlush = (io as unknown) === process ? installTelemetrySignalFlush(io, info) : undefined;
|
||||
const removeGlobalExceptionHandlers =
|
||||
(io as unknown) === process ? installGlobalExceptionHandlers(io, info) : undefined;
|
||||
try {
|
||||
return await runCommanderKtxCli(argv, io, deps, info, {
|
||||
runInit: runInitForCommander,
|
||||
});
|
||||
} finally {
|
||||
removeGlobalExceptionHandlers?.();
|
||||
removeSignalFlush?.();
|
||||
}
|
||||
return await runCommanderKtxCli(argv, io, deps, info, {
|
||||
runInit: runInitForCommander,
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,11 +16,7 @@ export function walkCommandTree(command: CommandUnknownOpts): CommandTreeNode {
|
|||
description: command.description(),
|
||||
aliases: command.aliases(),
|
||||
arguments: command.registeredArguments.map(formatArgumentDeclaration),
|
||||
// Internal commands (e.g. the shell-completion helper `__complete`) use a
|
||||
// `__` prefix and are omitted from the human-facing command tree.
|
||||
children: command.commands
|
||||
.filter((child) => !child.name().startsWith('__'))
|
||||
.map((child) => walkCommandTree(child)),
|
||||
children: command.commands.map((child) => walkCommandTree(child)),
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,44 +0,0 @@
|
|||
import { Argument, type Command } from '@commander-js/extra-typings';
|
||||
import type { KtxCliCommandContext } from '../cli-program.js';
|
||||
import { computeCompletions } from '../completion/complete-engine.js';
|
||||
import { completionScript } from '../completion/completion-scripts.js';
|
||||
import { createProjectCompletionProviders } from '../completion/dynamic-candidates.js';
|
||||
import { profileMark } from '../startup-profile.js';
|
||||
|
||||
profileMark('module:commands/completion-commands');
|
||||
|
||||
export function registerCompletionCommands(program: Command, context: KtxCliCommandContext): void {
|
||||
program
|
||||
.command('completion')
|
||||
.description('Print a shell completion script for ktx')
|
||||
.addArgument(new Argument('<shell>', 'Target shell').choices(['zsh', 'bash']))
|
||||
.addHelpText(
|
||||
'after',
|
||||
'\nEnable completion by adding the matching line to your shell startup file:\n' +
|
||||
' zsh: eval "$(ktx completion zsh)"\n' +
|
||||
' bash: eval "$(ktx completion bash)"\n',
|
||||
)
|
||||
.action((shell) => {
|
||||
context.io.stdout.write(completionScript(shell));
|
||||
});
|
||||
|
||||
// Hidden command invoked by the generated shell scripts. It must only ever
|
||||
// print newline-separated candidates to stdout and exit 0, so a TAB press is
|
||||
// never disrupted by an error, a telemetry notice, or a parse failure.
|
||||
program
|
||||
.command('__complete', { hidden: true })
|
||||
.argument('[words...]')
|
||||
.allowUnknownOption(true)
|
||||
.helpOption(false)
|
||||
.action(async (words: string[]) => {
|
||||
try {
|
||||
const candidates = await computeCompletions(program, words, createProjectCompletionProviders());
|
||||
if (candidates.length > 0) {
|
||||
context.io.stdout.write(`${candidates.join('\n')}\n`);
|
||||
}
|
||||
} catch {
|
||||
// Swallow: completion must never break the shell.
|
||||
}
|
||||
context.setExitCode(0);
|
||||
});
|
||||
}
|
||||
|
|
@ -29,6 +29,8 @@ export function registerIngestCommands(
|
|||
.usage('[options] [connectionId]')
|
||||
.argument('[connectionId]', 'Configured connection id to ingest (omit to ingest all)')
|
||||
.option('--all', 'Ingest all configured connections', false)
|
||||
.addOption(new Option('--fast', 'Use deterministic database schema ingest').conflicts('deep'))
|
||||
.addOption(new Option('--deep', 'Use AI-enriched database ingest').conflicts('fast'))
|
||||
.addOption(new Option('--query-history', 'Include database query-history usage patterns').conflicts('noQueryHistory'))
|
||||
.addOption(new Option('--no-query-history', 'Skip database query-history usage patterns'))
|
||||
.option('--query-history-window-days <days>', 'Query-history lookback window for this run', parsePositiveIntegerOption)
|
||||
|
|
@ -85,6 +87,8 @@ export function registerIngestCommands(
|
|||
all: selection.kind === 'all',
|
||||
json: options.json === true,
|
||||
inputMode: options.input === false ? 'disabled' : 'auto',
|
||||
...(options.fast === true ? { depth: 'fast' as const } : {}),
|
||||
...(options.deep === true ? { depth: 'deep' as const } : {}),
|
||||
queryHistory,
|
||||
...(options.queryHistoryWindowDays !== undefined ? { queryHistoryWindowDays: options.queryHistoryWindowDays } : {}),
|
||||
cliVersion: context.packageInfo.version,
|
||||
|
|
|
|||
|
|
@ -21,9 +21,9 @@ function isDebugEnabled(command: CommandWithGlobalOptions): boolean {
|
|||
}
|
||||
|
||||
export function registerWikiCommands(program: Command, context: KtxCliCommandContext): void {
|
||||
const wiki = program
|
||||
program
|
||||
.command('wiki')
|
||||
.description('List, search, or read local wiki pages')
|
||||
.description('List or search local wiki pages')
|
||||
.usage('[options] [query...]')
|
||||
.argument('[query...]', 'Search query; omit to list all pages')
|
||||
.option('--user-id <id>', 'Local user id', 'local')
|
||||
|
|
@ -76,18 +76,4 @@ export function registerWikiCommands(program: Command, context: KtxCliCommandCon
|
|||
});
|
||||
},
|
||||
);
|
||||
|
||||
wiki
|
||||
.command('read')
|
||||
.description('Read a wiki page file by key')
|
||||
.argument('<key>', 'Wiki page key')
|
||||
.action(async (key: string, _options, command) => {
|
||||
const parentOpts = command.parent?.opts() as { userId?: string } | undefined;
|
||||
await runKnowledgeArgs(context, {
|
||||
command: 'read',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
key,
|
||||
userId: parentOpts?.userId ?? 'local',
|
||||
});
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ function embeddingBackend(value: string): 'openai' | 'sentence-transformers' {
|
|||
}
|
||||
|
||||
function llmBackend(value: string): KtxSetupLlmBackend {
|
||||
if (value === 'anthropic' || value === 'vertex' || value === 'claude-code' || value === 'codex') {
|
||||
if (value === 'anthropic' || value === 'vertex' || value === 'claude-code') {
|
||||
return value;
|
||||
}
|
||||
throw new InvalidArgumentError(`invalid choice '${value}'`);
|
||||
|
|
@ -308,14 +308,9 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo
|
|||
.addOption(new Option('--source-git-url <url>', 'Git URL for dbt, MetricFlow, or LookML').hideHelp())
|
||||
.addOption(new Option('--source-branch <branch>', 'Git branch for source setup').hideHelp())
|
||||
.addOption(new Option('--source-subpath <path>', 'Repo subpath for source setup').hideHelp())
|
||||
.addOption(
|
||||
new Option(
|
||||
'--source-auth-token-ref <ref>',
|
||||
'env: or file: credential ref for source repo auth or Notion integration token',
|
||||
).hideHelp(),
|
||||
)
|
||||
.addOption(new Option('--source-auth-token-ref <ref>', 'env: or file: credential ref for source repo auth').hideHelp())
|
||||
.addOption(new Option('--source-url <url>', 'Source service URL for Metabase or Looker').hideHelp())
|
||||
.addOption(new Option('--source-api-key-ref <ref>', 'env: or file: API key ref for Metabase').hideHelp())
|
||||
.addOption(new Option('--source-api-key-ref <ref>', 'env: or file: API key ref for Metabase or Notion').hideHelp())
|
||||
.addOption(new Option('--source-client-id <id>', 'Looker client id').hideHelp())
|
||||
.addOption(new Option('--source-client-secret-ref <ref>', 'env: or file: Looker client secret ref').hideHelp())
|
||||
.addOption(new Option('--source-warehouse-connection-id <id>', 'Mapped warehouse connection id').hideHelp())
|
||||
|
|
@ -406,8 +401,6 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo
|
|||
}
|
||||
|
||||
const resolvedAgentScope = options.local ? 'local' : options.global ? 'global' : 'project';
|
||||
const debugEnabled =
|
||||
((command.optsWithGlobals ? command.optsWithGlobals() : command.opts()) as { debug?: unknown }).debug === true;
|
||||
await runSetupArgs(context, {
|
||||
command: 'run',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
|
|
@ -417,7 +410,6 @@ export function registerSetupCommands(program: Command, context: KtxCliCommandCo
|
|||
agentScope: resolvedAgentScope,
|
||||
skipAgents: options.skipAgents === true,
|
||||
inputMode: options.input === false ? 'disabled' : 'auto',
|
||||
...(debugEnabled ? { debug: true } : {}),
|
||||
yes: options.yes === true,
|
||||
cliVersion: context.packageInfo.version,
|
||||
...(options.llmBackend ? { llmBackend: options.llmBackend } : {}),
|
||||
|
|
|
|||
|
|
@ -94,28 +94,19 @@ export function registerSlCommands(program: Command, context: KtxCliCommandConte
|
|||
},
|
||||
);
|
||||
|
||||
sl.command('read')
|
||||
.description('Read a semantic-layer source YAML file')
|
||||
.argument('<sourceName>', 'Semantic-layer source name')
|
||||
.action(async (sourceName: string, _options, command) => {
|
||||
const parentOpts = command.parent?.opts() as { connectionId?: string } | undefined;
|
||||
await runSlArgs(context, {
|
||||
command: 'read',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId: parentOpts?.connectionId,
|
||||
sourceName,
|
||||
});
|
||||
});
|
||||
|
||||
sl.command('validate')
|
||||
.description('Validate a semantic-layer source')
|
||||
.description('Validate a semantic-layer source (set --connection-id on `ktx sl`)')
|
||||
.argument('<sourceName>', 'Semantic-layer source name')
|
||||
.action(async (sourceName: string, _options, command) => {
|
||||
const parentOpts = command.parent?.opts() as { connectionId?: string } | undefined;
|
||||
const connectionId = parentOpts?.connectionId;
|
||||
if (connectionId === undefined) {
|
||||
command.error("error: required option '--connection-id <id>' not specified");
|
||||
}
|
||||
await runSlArgs(context, {
|
||||
command: 'validate',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId: parentOpts?.connectionId,
|
||||
connectionId: connectionId as string,
|
||||
sourceName,
|
||||
});
|
||||
});
|
||||
|
|
@ -140,14 +131,10 @@ export function registerSlCommands(program: Command, context: KtxCliCommandConte
|
|||
throw new Error('sl query requires at least one --measure');
|
||||
}
|
||||
const parentOpts = command.parent?.opts() as { connectionId?: string } | undefined;
|
||||
const connectionId = parentOpts?.connectionId;
|
||||
if (connectionId === undefined) {
|
||||
command.error("error: required option '--connection-id <id>' not specified");
|
||||
}
|
||||
const args = slQueryCommandSchema.parse({
|
||||
command: 'query',
|
||||
projectDir: resolveCommandProjectDir(command),
|
||||
connectionId,
|
||||
connectionId: parentOpts?.connectionId,
|
||||
...(options.queryFile
|
||||
? { queryFile: options.queryFile }
|
||||
: {
|
||||
|
|
|
|||
|
|
@ -1,172 +0,0 @@
|
|||
import type { CommandUnknownOpts, Option } from '@commander-js/extra-typings';
|
||||
|
||||
/**
|
||||
* Dynamic completion candidates that depend on project state (semantic-layer
|
||||
* source names, wiki page keys, connection ids). Injected so the engine stays
|
||||
* pure and unit-testable without touching the filesystem.
|
||||
*/
|
||||
export interface CompletionProviders {
|
||||
/** Candidate operands for a positional argument of the active command path. */
|
||||
positionalCandidates(commandPath: string[], typedTokens: string[]): Promise<string[]>;
|
||||
/** Candidate values for an option that has no static `choices` (e.g. `--connection-id`). */
|
||||
optionValueCandidates(commandPath: string[], optionFlag: string, typedTokens: string[]): Promise<string[]>;
|
||||
}
|
||||
|
||||
interface ResolvedCommand {
|
||||
command: CommandUnknownOpts;
|
||||
/** Subcommand names from the root down to the active command (root name excluded). */
|
||||
commandPath: string[];
|
||||
}
|
||||
|
||||
function isHiddenCommand(command: CommandUnknownOpts): boolean {
|
||||
// Completion mirrors `ktx --help`: commands registered with `{ hidden: true }`
|
||||
// (the `__complete` helper and `mcp serve-internal`) are internal and must not
|
||||
// surface. Commander exposes this only through the private `_hidden` field its
|
||||
// own help renderer reads, so a name heuristic like a `__` prefix is not enough.
|
||||
return (command as { _hidden?: boolean })._hidden === true;
|
||||
}
|
||||
|
||||
function resolveCommand(program: CommandUnknownOpts, typedTokens: string[]): ResolvedCommand {
|
||||
let command: CommandUnknownOpts = program;
|
||||
const commandPath: string[] = [];
|
||||
for (let index = 0; index < typedTokens.length; index += 1) {
|
||||
const token = typedTokens[index];
|
||||
if (token.startsWith('-')) {
|
||||
// A value-taking option in the `--flag value` form consumes the next token
|
||||
// as its value, so skip that value before matching subcommands. Otherwise a
|
||||
// connection id like `query` would be resolved as the `sl query` subcommand
|
||||
// instead of being treated as the `--connection-id` value. The `--flag=value`
|
||||
// form carries its own value and consumes nothing extra.
|
||||
if (!token.includes('=')) {
|
||||
const option = findOption(command, token);
|
||||
if (option && !option.isBoolean()) {
|
||||
index += 1;
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const sub = command.commands.find((candidate) => candidate.name() === token || candidate.aliases().includes(token));
|
||||
if (sub) {
|
||||
command = sub;
|
||||
commandPath.push(sub.name());
|
||||
}
|
||||
}
|
||||
return { command, commandPath };
|
||||
}
|
||||
|
||||
function collectOptions(command: CommandUnknownOpts): Option[] {
|
||||
const options: Option[] = [];
|
||||
let current: CommandUnknownOpts | null = command;
|
||||
while (current) {
|
||||
options.push(...current.options);
|
||||
current = current.parent;
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
function findOption(command: CommandUnknownOpts, flag: string): Option | undefined {
|
||||
return collectOptions(command).find((option) => option.long === flag || option.short === flag);
|
||||
}
|
||||
|
||||
function isRepeatableOption(option: Option): boolean {
|
||||
// Variadic options, and options backed by a collector with an array default
|
||||
// (e.g. `--measure`/`--dimension`), may be supplied more than once.
|
||||
return option.variadic || Array.isArray(option.defaultValue);
|
||||
}
|
||||
|
||||
function flagCandidates(command: CommandUnknownOpts, typedTokens: string[]): string[] {
|
||||
const present = new Set(typedTokens.filter((token) => token.startsWith('-')));
|
||||
const candidates: string[] = [];
|
||||
for (const option of collectOptions(command)) {
|
||||
if (option.hidden || !option.long) {
|
||||
continue;
|
||||
}
|
||||
if (present.has(option.long) && !isRepeatableOption(option)) {
|
||||
continue;
|
||||
}
|
||||
candidates.push(option.long);
|
||||
}
|
||||
return candidates;
|
||||
}
|
||||
|
||||
async function optionValueCandidates(
|
||||
resolved: ResolvedCommand,
|
||||
option: Option,
|
||||
typedTokens: string[],
|
||||
providers: CompletionProviders,
|
||||
): Promise<string[]> {
|
||||
if (option.argChoices && option.argChoices.length > 0) {
|
||||
return option.argChoices;
|
||||
}
|
||||
return providers.optionValueCandidates(resolved.commandPath, option.long ?? option.name(), typedTokens);
|
||||
}
|
||||
|
||||
function dedupeSortFilter(candidates: string[], partial: string): string[] {
|
||||
const seen = new Set<string>();
|
||||
const matches: string[] = [];
|
||||
for (const candidate of candidates) {
|
||||
if (!candidate.startsWith(partial) || seen.has(candidate)) {
|
||||
continue;
|
||||
}
|
||||
seen.add(candidate);
|
||||
matches.push(candidate);
|
||||
}
|
||||
return matches.sort();
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute completion candidates for the partial last element of `words`
|
||||
* (everything the shell has on the line after `ktx`). The active command and
|
||||
* its flags are derived by walking the live Commander tree, so completion never
|
||||
* drifts from the real command structure.
|
||||
*/
|
||||
export async function computeCompletions(
|
||||
program: CommandUnknownOpts,
|
||||
words: string[],
|
||||
providers: CompletionProviders,
|
||||
): Promise<string[]> {
|
||||
const partial = words.length > 0 ? (words[words.length - 1] ?? '') : '';
|
||||
const typedTokens = words.slice(0, -1);
|
||||
const resolved = resolveCommand(program, typedTokens);
|
||||
|
||||
// (a) Option value via the `--opt=value` form.
|
||||
const equalsMatch = /^(--[^=]+)=(.*)$/.exec(partial);
|
||||
if (equalsMatch) {
|
||||
const [, flag, valuePartial] = equalsMatch;
|
||||
const option = findOption(resolved.command, flag);
|
||||
if (!option || option.isBoolean()) {
|
||||
return [];
|
||||
}
|
||||
const values = await optionValueCandidates(resolved, option, typedTokens, providers);
|
||||
return dedupeSortFilter(
|
||||
values.map((value) => `${flag}=${value}`),
|
||||
`${flag}=${valuePartial}`,
|
||||
);
|
||||
}
|
||||
|
||||
// (b) Option value via the `--opt value` form (previous token is a value-taking option).
|
||||
const previous = typedTokens[typedTokens.length - 1];
|
||||
if (previous && previous.startsWith('-') && !partial.startsWith('-')) {
|
||||
const option = findOption(resolved.command, previous);
|
||||
if (option && !option.isBoolean()) {
|
||||
return dedupeSortFilter(await optionValueCandidates(resolved, option, typedTokens, providers), partial);
|
||||
}
|
||||
}
|
||||
|
||||
// (c) Flag completion.
|
||||
if (partial.startsWith('-')) {
|
||||
return dedupeSortFilter(flagCandidates(resolved.command, typedTokens), partial);
|
||||
}
|
||||
|
||||
// (d) Positional: subcommand names union static argument choices union dynamic operand candidates.
|
||||
const candidates: string[] = resolved.command.commands
|
||||
.filter((sub) => !isHiddenCommand(sub))
|
||||
.map((sub) => sub.name());
|
||||
for (const argument of resolved.command.registeredArguments) {
|
||||
if (argument.argChoices) {
|
||||
candidates.push(...argument.argChoices);
|
||||
}
|
||||
}
|
||||
candidates.push(...(await providers.positionalCandidates(resolved.commandPath, typedTokens)));
|
||||
return dedupeSortFilter(candidates, partial);
|
||||
}
|
||||
|
|
@ -1,39 +0,0 @@
|
|||
// Static shell completion scripts emitted by `ktx completion <shell>`.
|
||||
//
|
||||
// Both scripts gather the words on the current command line (excluding the
|
||||
// leading `ktx`), append the partial word under the cursor, and delegate to the
|
||||
// hidden `ktx __complete` command, which prints newline-separated candidates.
|
||||
// All command/flag/entity knowledge lives in `ktx __complete` so these scripts
|
||||
// never have to encode the command tree.
|
||||
//
|
||||
// Lines are single-quoted JS strings so the shell `${...}` expansions are
|
||||
// emitted verbatim (a template literal would try to interpolate them).
|
||||
|
||||
const ZSH_SCRIPT = [
|
||||
'#compdef ktx',
|
||||
'_ktx() {',
|
||||
' local -a candidates',
|
||||
' local out',
|
||||
' out="$(ktx __complete -- "${words[@]:1:$((CURRENT-1))}" 2>/dev/null)" || return 0',
|
||||
' candidates=("${(@f)out}")',
|
||||
' compadd -- $candidates',
|
||||
'}',
|
||||
'compdef _ktx ktx',
|
||||
'',
|
||||
].join('\n');
|
||||
|
||||
const BASH_SCRIPT = [
|
||||
'_ktx() {',
|
||||
' local cur out',
|
||||
' cur="${COMP_WORDS[COMP_CWORD]}"',
|
||||
' out="$(ktx __complete -- "${COMP_WORDS[@]:1:COMP_CWORD}" 2>/dev/null)" || { COMPREPLY=(); return 0; }',
|
||||
" local IFS=$'\\n'",
|
||||
' COMPREPLY=($(compgen -W "${out}" -- "$cur"))',
|
||||
'}',
|
||||
'complete -F _ktx ktx',
|
||||
'',
|
||||
].join('\n');
|
||||
|
||||
export function completionScript(shell: 'zsh' | 'bash'): string {
|
||||
return shell === 'zsh' ? ZSH_SCRIPT : BASH_SCRIPT;
|
||||
}
|
||||
|
|
@ -1,103 +0,0 @@
|
|||
import { existsSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import type { KtxLocalProject } from '../context/project/project.js';
|
||||
import { resolveKtxProjectDir } from '../project-resolver.js';
|
||||
import type { CompletionProviders } from './complete-engine.js';
|
||||
|
||||
/** Extract an option value from already-typed tokens (`--flag value` or `--flag=value`). */
|
||||
function extractOptionValue(tokens: string[], flag: string): string | undefined {
|
||||
const prefix = `${flag}=`;
|
||||
for (let index = 0; index < tokens.length; index += 1) {
|
||||
const token = tokens[index];
|
||||
if (token === flag) {
|
||||
const next = tokens[index + 1];
|
||||
if (next !== undefined && !next.startsWith('-')) {
|
||||
return next;
|
||||
}
|
||||
} else if (token.startsWith(prefix)) {
|
||||
return token.slice(prefix.length);
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve and load the project the user is completing against. Honors a
|
||||
* `--project-dir` typed on the line, then `KTX_PROJECT_DIR`, then the nearest
|
||||
* `ktx.yaml`. Returns null (no completions) when there is no project, without
|
||||
* creating any files.
|
||||
*/
|
||||
async function loadCompletionProject(typedTokens: string[]): Promise<KtxLocalProject | null> {
|
||||
const explicitProjectDir = extractOptionValue(typedTokens, '--project-dir');
|
||||
const projectDir = resolveKtxProjectDir(explicitProjectDir !== undefined ? { explicitProjectDir } : {});
|
||||
if (!existsSync(join(projectDir, 'ktx.yaml'))) {
|
||||
return null;
|
||||
}
|
||||
const { loadKtxProject } = await import('../context/project/project.js');
|
||||
return loadKtxProject({ projectDir });
|
||||
}
|
||||
|
||||
async function sourceNames(typedTokens: string[]): Promise<string[]> {
|
||||
const project = await loadCompletionProject(typedTokens);
|
||||
if (!project) {
|
||||
return [];
|
||||
}
|
||||
const connectionId = extractOptionValue(typedTokens, '--connection-id');
|
||||
const { listLocalSlSources } = await import('../context/sl/local-sl.js');
|
||||
const summaries = await listLocalSlSources(project, connectionId !== undefined ? { connectionId } : {});
|
||||
return [...new Set(summaries.map((summary) => summary.name))];
|
||||
}
|
||||
|
||||
async function wikiPageKeys(typedTokens: string[]): Promise<string[]> {
|
||||
const project = await loadCompletionProject(typedTokens);
|
||||
if (!project) {
|
||||
return [];
|
||||
}
|
||||
const userId = extractOptionValue(typedTokens, '--user-id');
|
||||
const { listLocalKnowledgePageKeys } = await import('../context/wiki/local-knowledge.js');
|
||||
return listLocalKnowledgePageKeys(project, userId !== undefined ? { userId } : {});
|
||||
}
|
||||
|
||||
async function connectionIds(typedTokens: string[]): Promise<string[]> {
|
||||
const project = await loadCompletionProject(typedTokens);
|
||||
if (!project) {
|
||||
return [];
|
||||
}
|
||||
return Object.keys(project.config.connections).sort();
|
||||
}
|
||||
|
||||
/**
|
||||
* Project-backed completion providers. Every entry swallows its own errors so a
|
||||
* failed lookup never breaks the shell — completion degrades to commands/flags.
|
||||
*/
|
||||
export function createProjectCompletionProviders(): CompletionProviders {
|
||||
return {
|
||||
async positionalCandidates(commandPath, typedTokens) {
|
||||
try {
|
||||
const key = commandPath.join(' ');
|
||||
if (key === 'sl read' || key === 'sl validate') {
|
||||
return await sourceNames(typedTokens);
|
||||
}
|
||||
if (key === 'wiki read') {
|
||||
return await wikiPageKeys(typedTokens);
|
||||
}
|
||||
if (key === 'connection test' || key === 'ingest') {
|
||||
return await connectionIds(typedTokens);
|
||||
}
|
||||
return [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
},
|
||||
async optionValueCandidates(_commandPath, optionFlag, typedTokens) {
|
||||
try {
|
||||
if (optionFlag === '--connection-id' || optionFlag === '--connection') {
|
||||
return await connectionIds(typedTokens);
|
||||
}
|
||||
return [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
import type { KtxProjectConnectionConfig } from './context/project/config.js';
|
||||
|
||||
const KTX_DATABASE_DRIVER_IDS = new Set([
|
||||
'sqlite',
|
||||
'postgres',
|
||||
'mysql',
|
||||
'clickhouse',
|
||||
'sqlserver',
|
||||
'bigquery',
|
||||
'snowflake',
|
||||
]);
|
||||
|
||||
export function normalizeConnectionDriver(connection: KtxProjectConnectionConfig): string {
|
||||
return String(connection.driver ?? '')
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
}
|
||||
|
||||
export function isDatabaseDriver(driver: string): boolean {
|
||||
return KTX_DATABASE_DRIVER_IDS.has(driver.trim().toLowerCase());
|
||||
}
|
||||
|
|
@ -1,132 +0,0 @@
|
|||
import type { KtxCliIo } from './cli-runtime.js';
|
||||
import type { KtxSetupPromptOption } from './setup-prompts.js';
|
||||
|
||||
export type RecoveryOutcome = 'ready' | 'skip' | 'back' | 'failed';
|
||||
|
||||
/** @internal */
|
||||
export interface RecoveryAction {
|
||||
value: string;
|
||||
label: string;
|
||||
run: () => Promise<void>;
|
||||
}
|
||||
|
||||
export type ConfigureResult = 'configured' | 'back' | 'cancelled';
|
||||
|
||||
export type ValidateResult =
|
||||
| { status: 'ok' }
|
||||
| { status: 'back' }
|
||||
| { status: 'failed'; extraActions?: RecoveryAction[] };
|
||||
|
||||
export interface ConnectionRecoveryInput {
|
||||
label: string;
|
||||
interactive: boolean;
|
||||
allowSkip: boolean;
|
||||
io: KtxCliIo;
|
||||
prompts: {
|
||||
select(options: { message: string; options: KtxSetupPromptOption[] }): Promise<string>;
|
||||
};
|
||||
snapshot: () => Promise<() => Promise<void>>;
|
||||
configure: () => Promise<ConfigureResult>;
|
||||
validate: () => Promise<ValidateResult>;
|
||||
}
|
||||
|
||||
async function runRollbackOnce(input: {
|
||||
rollback: () => Promise<void>;
|
||||
state: { rolledBack: boolean };
|
||||
}): Promise<void> {
|
||||
if (input.state.rolledBack) {
|
||||
return;
|
||||
}
|
||||
input.state.rolledBack = true;
|
||||
await input.rollback();
|
||||
}
|
||||
|
||||
function recoveryOptions(input: {
|
||||
allowSkip: boolean;
|
||||
extraActions?: RecoveryAction[];
|
||||
}): KtxSetupPromptOption[] {
|
||||
return [
|
||||
{ value: 'retry', label: 'Retry connection test' },
|
||||
{ value: 're-enter', label: 'Re-enter connection details' },
|
||||
...(input.extraActions ?? []).map((action) => ({
|
||||
value: action.value,
|
||||
label: action.label,
|
||||
})),
|
||||
...(input.allowSkip ? [{ value: 'skip', label: 'Skip this connection' }] : []),
|
||||
{ value: 'back', label: 'Back' },
|
||||
];
|
||||
}
|
||||
|
||||
export async function runConnectionSetupWithRecovery(
|
||||
input: ConnectionRecoveryInput,
|
||||
): Promise<RecoveryOutcome> {
|
||||
const rollback = await input.snapshot();
|
||||
const rollbackState = { rolledBack: false };
|
||||
|
||||
const firstConfig = await input.configure();
|
||||
if (firstConfig === 'back') {
|
||||
await runRollbackOnce({ rollback, state: rollbackState });
|
||||
return 'back';
|
||||
}
|
||||
if (firstConfig === 'cancelled') {
|
||||
await runRollbackOnce({ rollback, state: rollbackState });
|
||||
return 'failed';
|
||||
}
|
||||
|
||||
let validation = await input.validate();
|
||||
while (validation.status !== 'ok') {
|
||||
if (validation.status === 'back') {
|
||||
await runRollbackOnce({ rollback, state: rollbackState });
|
||||
return 'back';
|
||||
}
|
||||
|
||||
if (!input.interactive) {
|
||||
return 'failed';
|
||||
}
|
||||
|
||||
const action = await input.prompts.select({
|
||||
message: `Connection setup failed for ${input.label}`,
|
||||
options: recoveryOptions({
|
||||
allowSkip: input.allowSkip,
|
||||
extraActions: validation.extraActions,
|
||||
}),
|
||||
});
|
||||
|
||||
if (action === 'back') {
|
||||
await runRollbackOnce({ rollback, state: rollbackState });
|
||||
return 'back';
|
||||
}
|
||||
if (action === 'skip' && input.allowSkip) {
|
||||
await runRollbackOnce({ rollback, state: rollbackState });
|
||||
return 'skip';
|
||||
}
|
||||
if (action === 're-enter') {
|
||||
const nextConfig = await input.configure();
|
||||
if (nextConfig === 'back') {
|
||||
await runRollbackOnce({ rollback, state: rollbackState });
|
||||
return 'back';
|
||||
}
|
||||
if (nextConfig === 'cancelled') {
|
||||
await runRollbackOnce({ rollback, state: rollbackState });
|
||||
return 'failed';
|
||||
}
|
||||
validation = await input.validate();
|
||||
continue;
|
||||
}
|
||||
if (action === 'retry') {
|
||||
validation = await input.validate();
|
||||
continue;
|
||||
}
|
||||
|
||||
const extraAction = validation.extraActions?.find((candidate) => candidate.value === action);
|
||||
if (extraAction) {
|
||||
await extraAction.run();
|
||||
validation = await input.validate();
|
||||
continue;
|
||||
}
|
||||
|
||||
validation = await input.validate();
|
||||
}
|
||||
|
||||
return 'ready';
|
||||
}
|
||||
|
|
@ -16,9 +16,8 @@ import { bold, dim, green, red, SYMBOLS } from './io/symbols.js';
|
|||
import { createKtxCliScanConnector } from './local-scan-connectors.js';
|
||||
import { profileMark } from './startup-profile.js';
|
||||
import { isDemoConnection } from './telemetry/demo-detect.js';
|
||||
import { emitTelemetryEvent, reportException } from './telemetry/index.js';
|
||||
import { collectTelemetryRedactionSecrets } from './telemetry/redaction-secrets.js';
|
||||
import { formatErrorDetail, scrubErrorClass } from './telemetry/scrubber.js';
|
||||
import { emitTelemetryEvent } from './telemetry/index.js';
|
||||
import { scrubErrorClass } from './telemetry/scrubber.js';
|
||||
|
||||
profileMark('module:connection');
|
||||
|
||||
|
|
@ -75,12 +74,6 @@ async function testNativeConnection(
|
|||
}
|
||||
const result = await connector.testConnection();
|
||||
if (!result.success) {
|
||||
// Re-throw the driver's original error so connection_test telemetry records
|
||||
// its real class (e.g. ConnectionError) and code (e.g. ELOGIN) instead of
|
||||
// collapsing every native failure to a generic Error with no code.
|
||||
if (result.cause instanceof Error) {
|
||||
throw result.cause;
|
||||
}
|
||||
throw new Error(result.error ?? 'connection test failed');
|
||||
}
|
||||
return { driver: connector.driver };
|
||||
|
|
@ -311,7 +304,6 @@ async function emitConnectionTest(input: {
|
|||
io: KtxCliIo;
|
||||
}): Promise<void> {
|
||||
const errorClass = input.error ? scrubErrorClass(input.error) : undefined;
|
||||
const errorDetail = input.error ? formatErrorDetail(input.error) : undefined;
|
||||
await emitTelemetryEvent({
|
||||
name: 'connection_test',
|
||||
projectDir: input.project.projectDir,
|
||||
|
|
@ -322,24 +314,8 @@ async function emitConnectionTest(input: {
|
|||
outcome: input.outcome,
|
||||
durationMs: input.durationMs,
|
||||
...(errorClass ? { errorClass } : {}),
|
||||
...(errorDetail ? { errorDetail } : {}),
|
||||
},
|
||||
});
|
||||
if (input.error) {
|
||||
await reportException({
|
||||
error: input.error,
|
||||
context: { source: 'connection test', handled: true, fatal: false },
|
||||
projectDir: input.project.projectDir,
|
||||
io: input.io,
|
||||
redactionSecrets: await collectTelemetryRedactionSecrets({
|
||||
project: input.project,
|
||||
connectionId: input.connectionId,
|
||||
includeLlm: false,
|
||||
includeEmbeddings: false,
|
||||
env: process.env,
|
||||
}),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function visualWidth(text: string): number {
|
||||
|
|
|
|||
|
|
@ -5,9 +5,7 @@ import { assertReadOnlySql, limitSqlForExecution } from '../../context/connectio
|
|||
import { tryConstraintQuery } from '../../context/scan/constraint-discovery.js';
|
||||
import { scopedTableNames } from '../../context/scan/table-ref.js';
|
||||
import {
|
||||
connectorTestFailure,
|
||||
createKtxConnectorCapabilities,
|
||||
type KtxConnectorTestResult,
|
||||
type KtxColumnSampleInput,
|
||||
type KtxColumnSampleResult,
|
||||
type KtxColumnStatsInput,
|
||||
|
|
@ -322,7 +320,7 @@ export class KtxBigQueryScanConnector implements KtxScanConnector {
|
|||
this.id = `bigquery:${options.connectionId}`;
|
||||
}
|
||||
|
||||
async testConnection(): Promise<KtxConnectorTestResult> {
|
||||
async testConnection(): Promise<{ success: boolean; error?: string }> {
|
||||
try {
|
||||
const client = this.getClient();
|
||||
await client.getDatasets({ maxResults: 1 });
|
||||
|
|
@ -331,7 +329,7 @@ export class KtxBigQueryScanConnector implements KtxScanConnector {
|
|||
}
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
return connectorTestFailure(error);
|
||||
return { success: false, error: error instanceof Error ? error.message : String(error) };
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { createClient } from '@clickhouse/client';
|
||||
import { getDialectForDriver } from '../../context/connections/dialects.js';
|
||||
import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js';
|
||||
import { connectorTestFailure, createKtxConnectorCapabilities, type KtxConnectorTestResult, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaColumn, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableRef, type KtxTableSampleInput, type KtxTableListEntry, type KtxTableSampleResult } from '../../context/scan/types.js';
|
||||
import { createKtxConnectorCapabilities, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaColumn, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableRef, type KtxTableSampleInput, type KtxTableListEntry, type KtxTableSampleResult } from '../../context/scan/types.js';
|
||||
import { scopedTableNames } from '../../context/scan/table-ref.js';
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { Agent as HttpsAgent } from 'node:https';
|
||||
|
|
@ -317,12 +317,12 @@ export class KtxClickHouseScanConnector implements KtxScanConnector {
|
|||
this.id = `clickhouse:${options.connectionId}`;
|
||||
}
|
||||
|
||||
async testConnection(): Promise<KtxConnectorTestResult> {
|
||||
async testConnection(): Promise<{ success: boolean; error?: string }> {
|
||||
try {
|
||||
await this.query('SELECT 1');
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
return connectorTestFailure(error);
|
||||
return { success: false, error: error instanceof Error ? error.message : String(error) };
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -11,9 +11,7 @@ import {
|
|||
} from '../../context/scan/constraint-discovery.js';
|
||||
import { scopedTableNames } from '../../context/scan/table-ref.js';
|
||||
import {
|
||||
connectorTestFailure,
|
||||
createKtxConnectorCapabilities,
|
||||
type KtxConnectorTestResult,
|
||||
type KtxColumnSampleInput,
|
||||
type KtxColumnSampleResult,
|
||||
type KtxColumnStatsInput,
|
||||
|
|
@ -415,12 +413,12 @@ export class KtxMysqlScanConnector implements KtxScanConnector {
|
|||
this.id = `mysql:${options.connectionId}`;
|
||||
}
|
||||
|
||||
async testConnection(): Promise<KtxConnectorTestResult> {
|
||||
async testConnection(): Promise<{ success: boolean; error?: string }> {
|
||||
try {
|
||||
await this.query('SELECT 1');
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
return connectorTestFailure(error);
|
||||
return { success: false, error: error instanceof Error ? error.message : String(error) };
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -6,9 +6,7 @@ import { assertReadOnlySql, limitSqlForExecution } from '../../context/connectio
|
|||
import { tryConstraintQuery } from '../../context/scan/constraint-discovery.js';
|
||||
import { scopedTableNames } from '../../context/scan/table-ref.js';
|
||||
import {
|
||||
connectorTestFailure,
|
||||
createKtxConnectorCapabilities,
|
||||
type KtxConnectorTestResult,
|
||||
type KtxColumnSampleInput,
|
||||
type KtxColumnSampleResult,
|
||||
type KtxColumnStatsInput,
|
||||
|
|
@ -444,12 +442,12 @@ export class KtxPostgresScanConnector implements KtxScanConnector {
|
|||
this.id = `postgres:${options.connectionId}`;
|
||||
}
|
||||
|
||||
async testConnection(): Promise<KtxConnectorTestResult> {
|
||||
async testConnection(): Promise<{ success: boolean; error?: string }> {
|
||||
try {
|
||||
await this.query('SELECT 1');
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
return connectorTestFailure(error);
|
||||
return { success: false, error: error instanceof Error ? error.message : String(error) };
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -7,9 +7,7 @@ import { assertReadOnlySql, limitSqlForExecution } from '../../context/connectio
|
|||
import { tryConstraintQuery } from '../../context/scan/constraint-discovery.js';
|
||||
import { scopedTableNames } from '../../context/scan/table-ref.js';
|
||||
import {
|
||||
connectorTestFailure,
|
||||
createKtxConnectorCapabilities,
|
||||
type KtxConnectorTestResult,
|
||||
type KtxColumnSampleInput,
|
||||
type KtxColumnSampleResult,
|
||||
type KtxColumnStatsInput,
|
||||
|
|
@ -466,7 +464,7 @@ class SnowflakeSdkDriver implements KtxSnowflakeDriver {
|
|||
await this.query('SELECT 1');
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
return connectorTestFailure(error);
|
||||
return { success: false, error: error instanceof Error ? error.message : String(error) };
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -575,7 +573,7 @@ export class KtxSnowflakeScanConnector implements KtxScanConnector {
|
|||
}
|
||||
}
|
||||
|
||||
async testConnection(): Promise<KtxConnectorTestResult> {
|
||||
async testConnection(): Promise<{ success: boolean; error?: string }> {
|
||||
return this.getDriver().test();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import { fileURLToPath } from 'node:url';
|
|||
import { getDialectForDriver } from '../../context/connections/dialects.js';
|
||||
import { assertReadOnlySql, limitSqlForExecution } from '../../context/connections/read-only-sql.js';
|
||||
import { normalizeQueryRows } from '../../context/connections/query-executor.js';
|
||||
import { connectorTestFailure, createKtxConnectorCapabilities, type KtxConnectorTestResult, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaForeignKey, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableListEntry, type KtxTableRef, type KtxTableSampleInput, type KtxTableSampleResult } from '../../context/scan/types.js';
|
||||
import { createKtxConnectorCapabilities, type KtxColumnSampleInput, type KtxColumnSampleResult, type KtxColumnStatsInput, type KtxColumnStatsResult, type KtxQueryResult, type KtxReadOnlyQueryInput, type KtxScanConnector, type KtxScanContext, type KtxScanInput, type KtxSchemaForeignKey, type KtxSchemaSnapshot, type KtxSchemaTable, type KtxTableListEntry, type KtxTableRef, type KtxTableSampleInput, type KtxTableSampleResult } from '../../context/scan/types.js';
|
||||
import { scopedTableNames } from '../../context/scan/table-ref.js';
|
||||
|
||||
export interface KtxSqliteConnectionConfig {
|
||||
|
|
@ -167,7 +167,7 @@ export class KtxSqliteScanConnector implements KtxScanConnector {
|
|||
this.id = `sqlite:${options.connectionId}`;
|
||||
}
|
||||
|
||||
async testConnection(): Promise<KtxConnectorTestResult> {
|
||||
async testConnection(): Promise<{ success: boolean; error?: string }> {
|
||||
try {
|
||||
if (!existsSync(this.dbPath) || !statSync(this.dbPath).isFile()) {
|
||||
return { success: false, error: `File not found: ${this.dbPath}` };
|
||||
|
|
@ -175,7 +175,7 @@ export class KtxSqliteScanConnector implements KtxScanConnector {
|
|||
this.database().prepare('SELECT 1').get();
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
return connectorTestFailure(error);
|
||||
return { success: false, error: error instanceof Error ? error.message : String(error) };
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -3,9 +3,7 @@ import { getDialectForDriver } from '../../context/connections/dialects.js';
|
|||
import { tryConstraintQuery } from '../../context/scan/constraint-discovery.js';
|
||||
import { scopedTableNames } from '../../context/scan/table-ref.js';
|
||||
import {
|
||||
connectorTestFailure,
|
||||
createKtxConnectorCapabilities,
|
||||
type KtxConnectorTestResult,
|
||||
type KtxColumnSampleInput,
|
||||
type KtxColumnSampleResult,
|
||||
type KtxColumnStatsInput,
|
||||
|
|
@ -386,12 +384,12 @@ export class KtxSqlServerScanConnector implements KtxScanConnector {
|
|||
this.id = `sqlserver:${options.connectionId}`;
|
||||
}
|
||||
|
||||
async testConnection(): Promise<KtxConnectorTestResult> {
|
||||
async testConnection(): Promise<{ success: boolean; error?: string }> {
|
||||
try {
|
||||
await this.query('SELECT 1');
|
||||
return { success: true };
|
||||
} catch (error) {
|
||||
return connectorTestFailure(error);
|
||||
return { success: false, error: error instanceof Error ? error.message : String(error) };
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
import type { KtxProgressPort, KtxProgressUpdateOptions } from './context/scan/types.js';
|
||||
import type { KtxCliIo } from './index.js';
|
||||
import type { KtxIngestProgressUpdate } from './ingest.js';
|
||||
import type { KtxManagedPythonInstallPolicy } from './managed-python-command.js';
|
||||
import { publicDatabaseIngestMessage, publicQueryHistoryMessage } from './public-ingest-copy.js';
|
||||
import type {
|
||||
KtxPublicIngestArgs,
|
||||
KtxPublicIngestDeps,
|
||||
|
|
@ -8,8 +10,7 @@ import type {
|
|||
KtxPublicIngestProject,
|
||||
KtxPublicIngestTargetResult,
|
||||
} from './public-ingest.js';
|
||||
import { buildPublicIngestPlan, executePublicIngestTarget, publicProgressMessage } from './public-ingest.js';
|
||||
import { createAggregateProgressPort } from './progress-port-adapter.js';
|
||||
import { buildPublicIngestPlan, executePublicIngestTarget } from './public-ingest.js';
|
||||
import { formatDuration } from './demo-metrics.js';
|
||||
import { profileMark } from './startup-profile.js';
|
||||
|
||||
|
|
@ -87,6 +88,7 @@ export interface ContextBuildArgs {
|
|||
targetConnectionId?: string;
|
||||
all?: boolean;
|
||||
entrypoint?: 'setup' | 'ingest';
|
||||
depth?: Extract<KtxPublicIngestArgs, { command: 'run' }>['depth'];
|
||||
queryHistory?: Extract<KtxPublicIngestArgs, { command: 'run' }>['queryHistory'];
|
||||
queryHistoryWindowDays?: number;
|
||||
scanMode?: Extract<KtxPublicIngestArgs, { command: 'run' }>['scanMode'];
|
||||
|
|
@ -369,17 +371,19 @@ function retryCommand(input: {
|
|||
projectDir?: string;
|
||||
entrypoint?: 'setup' | 'ingest';
|
||||
connectionId?: string;
|
||||
depth?: 'fast' | 'deep';
|
||||
queryHistory?: boolean;
|
||||
queryHistoryWindowDays?: number;
|
||||
}): string {
|
||||
const projectPart = input.projectDir ? ` --project-dir ${input.projectDir}` : '';
|
||||
if (input.entrypoint === 'ingest' && input.connectionId) {
|
||||
const depthPart = input.depth ? ` --${input.depth}` : '';
|
||||
const queryHistoryPart = input.queryHistory ? ' --query-history' : '';
|
||||
const windowPart =
|
||||
input.queryHistory && input.queryHistoryWindowDays !== undefined
|
||||
? ` --query-history-window-days ${input.queryHistoryWindowDays}`
|
||||
: '';
|
||||
return `ktx ingest ${input.connectionId}${projectPart}${queryHistoryPart}${windowPart}`;
|
||||
return `ktx ingest ${input.connectionId}${projectPart}${depthPart}${queryHistoryPart}${windowPart}`;
|
||||
}
|
||||
return input.projectDir ? `ktx setup --project-dir ${input.projectDir}` : 'ktx setup';
|
||||
}
|
||||
|
|
@ -742,6 +746,7 @@ function appendRetryIfNeeded(input: {
|
|||
projectDir: input.projectDir,
|
||||
entrypoint: input.entrypoint,
|
||||
connectionId: input.target.connectionId,
|
||||
depth: input.target.databaseDepth,
|
||||
queryHistory: input.target.queryHistory?.enabled === true,
|
||||
queryHistoryWindowDays: input.target.queryHistory?.windowDays,
|
||||
})}`;
|
||||
|
|
@ -764,6 +769,7 @@ function failureTextForTarget(input: {
|
|||
projectDir: input.projectDir,
|
||||
entrypoint: input.entrypoint,
|
||||
connectionId: input.target.connectionId,
|
||||
depth: input.target.databaseDepth,
|
||||
queryHistory: input.target.queryHistory?.enabled === true,
|
||||
queryHistoryWindowDays: input.target.queryHistory?.windowDays,
|
||||
})}`,
|
||||
|
|
@ -778,6 +784,7 @@ function failureTextForTarget(input: {
|
|||
projectDir: input.projectDir,
|
||||
entrypoint: input.entrypoint,
|
||||
connectionId: input.target.connectionId,
|
||||
depth: input.target.databaseDepth,
|
||||
queryHistory: input.target.queryHistory?.enabled === true,
|
||||
queryHistoryWindowDays: input.target.queryHistory?.windowDays,
|
||||
})}`,
|
||||
|
|
@ -809,6 +816,17 @@ export function initViewState(targets: KtxPublicIngestPlanTarget[]): ContextBuil
|
|||
};
|
||||
}
|
||||
|
||||
function publicProgressMessage(message: string, target: KtxPublicIngestPlanTarget): string {
|
||||
let current = message;
|
||||
if (target.operation === 'database-ingest') {
|
||||
current = publicDatabaseIngestMessage(current);
|
||||
}
|
||||
if (target.steps.includes('query-history')) {
|
||||
current = publicQueryHistoryMessage(current, target.connectionId);
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
function formatProgressDetail(
|
||||
update: Pick<KtxIngestProgressUpdate, 'percent' | 'message'>,
|
||||
target: KtxPublicIngestPlanTarget,
|
||||
|
|
@ -817,6 +835,29 @@ function formatProgressDetail(
|
|||
return `[${percent}%] ${publicProgressMessage(update.message, target)}`;
|
||||
}
|
||||
|
||||
function createContextBuildProgressPort(
|
||||
onProgress: (update: KtxIngestProgressUpdate) => void,
|
||||
state: { progress: number } = { progress: 0 },
|
||||
start = 0,
|
||||
weight = 1,
|
||||
): KtxProgressPort {
|
||||
return {
|
||||
async update(value: number, message?: string, options?: KtxProgressUpdateOptions): Promise<void> {
|
||||
const absoluteValue = start + Math.max(0, Math.min(1, value)) * weight;
|
||||
state.progress = Math.max(state.progress, Math.min(1, absoluteValue));
|
||||
if (!message) return;
|
||||
onProgress({
|
||||
percent: Math.max(0, Math.min(100, Math.round(state.progress * 100))),
|
||||
message,
|
||||
...(options?.transient !== undefined ? { transient: options.transient } : {}),
|
||||
});
|
||||
},
|
||||
startPhase(phaseWeight: number): KtxProgressPort {
|
||||
return createContextBuildProgressPort(onProgress, state, state.progress, weight * phaseWeight);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export async function runContextBuild(
|
||||
project: KtxPublicIngestProject,
|
||||
args: ContextBuildArgs,
|
||||
|
|
@ -827,6 +868,7 @@ export async function runContextBuild(
|
|||
projectDir: args.projectDir,
|
||||
...(args.targetConnectionId ? { targetConnectionId: args.targetConnectionId } : {}),
|
||||
all: args.all ?? true,
|
||||
...(args.depth ? { depth: args.depth } : {}),
|
||||
...(args.queryHistory ? { queryHistory: args.queryHistory } : {}),
|
||||
...(args.queryHistoryWindowDays !== undefined ? { queryHistoryWindowDays: args.queryHistoryWindowDays } : {}),
|
||||
...(args.scanMode ? { scanMode: args.scanMode } : {}),
|
||||
|
|
@ -893,6 +935,7 @@ export async function runContextBuild(
|
|||
all: args.all ?? true,
|
||||
json: false,
|
||||
inputMode: args.inputMode,
|
||||
...(args.depth ? { depth: args.depth } : {}),
|
||||
...(args.queryHistory ? { queryHistory: args.queryHistory } : {}),
|
||||
...(args.queryHistoryWindowDays !== undefined ? { queryHistoryWindowDays: args.queryHistoryWindowDays } : {}),
|
||||
...(args.scanMode ? { scanMode: args.scanMode } : {}),
|
||||
|
|
@ -987,7 +1030,7 @@ export async function runContextBuild(
|
|||
};
|
||||
|
||||
const progressDeps: KtxPublicIngestDeps = {
|
||||
scanProgress: createAggregateProgressPort(updateSchemaPhase),
|
||||
scanProgress: createContextBuildProgressPort(updateSchemaPhase),
|
||||
ingestProgress: updateIngestPhase,
|
||||
runtimeIo: io,
|
||||
onPhaseStart,
|
||||
|
|
@ -997,7 +1040,7 @@ export async function runContextBuild(
|
|||
let result: KtxPublicIngestTargetResult | null = null;
|
||||
let thrownError: unknown = null;
|
||||
try {
|
||||
result = await execTarget(targetState.target, runArgs, capture.io, progressDeps, project);
|
||||
result = await execTarget(targetState.target, runArgs, capture.io, progressDeps);
|
||||
} catch (error) {
|
||||
thrownError = error;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,39 +0,0 @@
|
|||
/** @internal */
|
||||
export function createAbortError(message = 'Aborted'): DOMException {
|
||||
return new DOMException(message, 'AbortError');
|
||||
}
|
||||
|
||||
export function isAbortError(error: unknown): boolean {
|
||||
if (error instanceof DOMException && error.name === 'AbortError') {
|
||||
return true;
|
||||
}
|
||||
if (!error || typeof error !== 'object') {
|
||||
return false;
|
||||
}
|
||||
const record = error as { name?: unknown; code?: unknown };
|
||||
return record.name === 'AbortError' || record.code === 'ABORT_ERR';
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function throwIfAborted(signal?: AbortSignal): void {
|
||||
if (signal?.aborted) {
|
||||
throw createAbortError();
|
||||
}
|
||||
}
|
||||
|
||||
export function linkAbortSignal(parent?: AbortSignal): { controller: AbortController; dispose: () => void } {
|
||||
const controller = new AbortController();
|
||||
if (!parent) {
|
||||
return { controller, dispose: () => undefined };
|
||||
}
|
||||
if (parent.aborted) {
|
||||
controller.abort(createAbortError());
|
||||
return { controller, dispose: () => undefined };
|
||||
}
|
||||
const onAbort = () => controller.abort(createAbortError());
|
||||
parent.addEventListener('abort', onAbort, { once: true });
|
||||
return {
|
||||
controller,
|
||||
dispose: () => parent.removeEventListener('abort', onAbort),
|
||||
};
|
||||
}
|
||||
|
|
@ -200,78 +200,27 @@ export class BigQueryHistoricSqlQueryHistoryReader {
|
|||
config: HistoricSqlUnifiedPullConfig,
|
||||
): AsyncIterable<AggregatedTemplate> {
|
||||
const sql = `
|
||||
WITH filtered_jobs AS (
|
||||
SELECT
|
||||
COALESCE(query_info.query_hashes.normalized_literals, TO_HEX(SHA256(query))) AS template_id,
|
||||
query,
|
||||
user_email,
|
||||
creation_time,
|
||||
end_time,
|
||||
error_result
|
||||
FROM ${this.viewPath}
|
||||
WHERE job_type = 'QUERY'
|
||||
AND statement_type IN ('SELECT', 'MERGE')
|
||||
AND creation_time >= ${timestampExpression(window.start)}
|
||||
AND creation_time < ${timestampExpression(window.end)}
|
||||
AND query IS NOT NULL
|
||||
),
|
||||
template_stats AS (
|
||||
SELECT
|
||||
template_id,
|
||||
MIN(query) AS canonical_sql,
|
||||
COUNT(*) AS executions,
|
||||
COUNT(DISTINCT user_email) AS distinct_users,
|
||||
MIN(creation_time) AS first_seen,
|
||||
MAX(creation_time) AS last_seen,
|
||||
APPROX_QUANTILES(TIMESTAMP_DIFF(end_time, creation_time, MILLISECOND), 100)[OFFSET(50)] AS p50_ms,
|
||||
APPROX_QUANTILES(TIMESTAMP_DIFF(end_time, creation_time, MILLISECOND), 100)[OFFSET(95)] AS p95_ms,
|
||||
SAFE_DIVIDE(COUNTIF(error_result IS NOT NULL), COUNT(*)) AS error_rate,
|
||||
CAST(NULL AS INT64) AS rows_produced
|
||||
FROM filtered_jobs
|
||||
GROUP BY template_id
|
||||
HAVING COUNT(*) >= ${config.minExecutions}
|
||||
),
|
||||
template_users AS (
|
||||
SELECT
|
||||
template_id,
|
||||
user_email AS user,
|
||||
COUNT(*) AS executions,
|
||||
MAX(creation_time) AS last_seen
|
||||
FROM filtered_jobs
|
||||
GROUP BY template_id, user_email
|
||||
)
|
||||
SELECT
|
||||
stats.template_id,
|
||||
stats.canonical_sql,
|
||||
stats.executions,
|
||||
stats.distinct_users,
|
||||
stats.first_seen,
|
||||
stats.last_seen,
|
||||
stats.p50_ms,
|
||||
stats.p95_ms,
|
||||
stats.error_rate,
|
||||
stats.rows_produced,
|
||||
TO_JSON_STRING(
|
||||
ARRAY_AGG(
|
||||
STRUCT(users.user AS user, users.executions AS executions)
|
||||
ORDER BY users.executions DESC, users.last_seen DESC
|
||||
)
|
||||
) AS top_users
|
||||
FROM template_stats AS stats
|
||||
JOIN template_users AS users
|
||||
ON users.template_id = stats.template_id
|
||||
GROUP BY
|
||||
stats.template_id,
|
||||
stats.canonical_sql,
|
||||
stats.executions,
|
||||
stats.distinct_users,
|
||||
stats.first_seen,
|
||||
stats.last_seen,
|
||||
stats.p50_ms,
|
||||
stats.p95_ms,
|
||||
stats.error_rate,
|
||||
stats.rows_produced
|
||||
ORDER BY stats.executions DESC`.trim();
|
||||
query_hash AS template_id,
|
||||
MIN(query) AS canonical_sql,
|
||||
COUNT(*) AS executions,
|
||||
COUNT(DISTINCT user_email) AS distinct_users,
|
||||
MIN(creation_time) AS first_seen,
|
||||
MAX(creation_time) AS last_seen,
|
||||
APPROX_QUANTILES(TIMESTAMP_DIFF(end_time, creation_time, MILLISECOND), 100)[OFFSET(50)] AS p50_ms,
|
||||
APPROX_QUANTILES(TIMESTAMP_DIFF(end_time, creation_time, MILLISECOND), 100)[OFFSET(95)] AS p95_ms,
|
||||
SAFE_DIVIDE(COUNTIF(error_result IS NOT NULL), COUNT(*)) AS error_rate,
|
||||
CAST(NULL AS INT64) AS rows_produced,
|
||||
TO_JSON_STRING(ARRAY_AGG(STRUCT(user_email AS user, 1 AS executions) ORDER BY creation_time DESC LIMIT 5)) AS top_users
|
||||
FROM ${this.viewPath}
|
||||
WHERE job_type = 'QUERY'
|
||||
AND statement_type IN ('SELECT', 'MERGE')
|
||||
AND creation_time >= ${timestampExpression(window.start)}
|
||||
AND creation_time < ${timestampExpression(window.end)}
|
||||
AND query IS NOT NULL
|
||||
GROUP BY query_hash
|
||||
HAVING COUNT(*) >= ${config.minExecutions}
|
||||
ORDER BY executions DESC`.trim();
|
||||
const result = await queryClient(client).executeQuery(sql);
|
||||
if (result.error) {
|
||||
throw grantsError(result.error);
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
import { createHash } from 'node:crypto';
|
||||
import { readFile, readdir } from 'node:fs/promises';
|
||||
import { join, relative } from 'node:path';
|
||||
import { tableRefKey } from '../../../scan/table-ref.js';
|
||||
import type { ChunkResult, DiffSet, ScopeDescriptor, WorkUnit } from '../../types.js';
|
||||
import { isHistoricSqlPatternInputShardPath } from './pattern-inputs.js';
|
||||
import { stagedManifestSchema, stagedPatternsInputSchema, stagedTableInputSchema } from './types.js';
|
||||
|
|
@ -38,7 +37,7 @@ export async function chunkHistoricSqlUnifiedStagedDir(stagedDir: string, diffSe
|
|||
}
|
||||
const table = stagedTableInputSchema.parse(await readJson(stagedDir, path));
|
||||
workUnits.push({
|
||||
unitKey: `historic-sql-table-${safeUnitKey(tableRefKey(table.tableRef))}`,
|
||||
unitKey: `historic-sql-table-${safeUnitKey(table.table)}`,
|
||||
displayLabel: `Historic SQL usage: ${table.table}`,
|
||||
rawFiles: [path],
|
||||
dependencyPaths: ['manifest.json'],
|
||||
|
|
|
|||
|
|
@ -26,21 +26,6 @@ export function isQueryHistoryEnabled(connection: unknown): boolean {
|
|||
return queryHistoryRecord(connection)?.enabled === true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves the query-history dialect from the connection's driver capability
|
||||
* alone, ignoring whether query history is enabled in ktx.yaml. Use this on the
|
||||
* adapter-registration path when query history has been explicitly requested
|
||||
* for the run (e.g. via `--query-history`, which is itself the opt-in): the
|
||||
* persisted `context.queryHistory.enabled` flag must not gate registration.
|
||||
* Returns null when the connection's driver has no query-history reader.
|
||||
*/
|
||||
export function historicSqlDialectForConnectionDriver(connection: unknown): HistoricSqlDialect | null {
|
||||
const conn = recordOrNull(connection);
|
||||
const driver = String(conn?.driver ?? '').toLowerCase();
|
||||
const registration = getDriverRegistration(driver);
|
||||
return registration?.hasHistoricSqlReader ? historicSqlDialectForDriver(registration.driver) : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves the query-history dialect for a connection. Returns null when
|
||||
* query history is disabled, or when the connection's driver has no
|
||||
|
|
@ -50,5 +35,8 @@ export function queryHistoryDialectForConnection(connection: unknown): HistoricS
|
|||
if (!isQueryHistoryEnabled(connection)) {
|
||||
return null;
|
||||
}
|
||||
return historicSqlDialectForConnectionDriver(connection);
|
||||
const conn = recordOrNull(connection);
|
||||
const driver = String(conn?.driver ?? '').toLowerCase();
|
||||
const registration = getDriverRegistration(driver);
|
||||
return registration?.hasHistoricSqlReader ? historicSqlDialectForDriver(registration.driver) : null;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
import { Buffer } from 'node:buffer';
|
||||
import { tableRefKey } from '../../../scan/table-ref.js';
|
||||
import type { StagedPatternsInput } from './types.js';
|
||||
|
||||
const HISTORIC_SQL_PATTERN_WORKUNIT_DIR = 'patterns-input';
|
||||
|
|
@ -45,16 +44,11 @@ function sortedAuditTemplates(templates: readonly PatternTemplate[]): PatternTem
|
|||
function sortedPatternCandidates(templates: readonly PatternTemplate[]): PatternTemplate[] {
|
||||
return [...templates]
|
||||
.filter((template) => template.tablesTouched.length >= 2)
|
||||
.map((template) => ({
|
||||
...template,
|
||||
tablesTouched: [...template.tablesTouched].sort((left, right) => tableRefKey(left).localeCompare(tableRefKey(right))),
|
||||
}))
|
||||
.map((template) => ({ ...template, tablesTouched: [...template.tablesTouched].sort() }))
|
||||
.sort((left, right) => {
|
||||
const cardinality = right.tablesTouched.length - left.tablesTouched.length;
|
||||
if (cardinality !== 0) return cardinality;
|
||||
const leftSignature = left.tablesTouched.map(tableRefKey).join('\0');
|
||||
const rightSignature = right.tablesTouched.map(tableRefKey).join('\0');
|
||||
const tableSignature = leftSignature.localeCompare(rightSignature);
|
||||
const tableSignature = left.tablesTouched.join('\0').localeCompare(right.tablesTouched.join('\0'));
|
||||
if (tableSignature !== 0) return tableSignature;
|
||||
return left.id.localeCompare(right.id);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,283 +0,0 @@
|
|||
import { z } from 'zod';
|
||||
import type { KtxLlmRuntimePort } from '../../../../context/llm/runtime-port.js';
|
||||
import type { SqlAnalysisPort } from '../../../../context/sql-analysis/ports.js';
|
||||
import { tableRefKey } from '../../../scan/table-ref.js';
|
||||
import type { KtxTableRef } from '../../../scan/types.js';
|
||||
import { bucketDistinctUsers, bucketExecutions, bucketRecency } from './buckets.js';
|
||||
import {
|
||||
compileHistoricSqlRedactionPatterns,
|
||||
redactHistoricSqlText,
|
||||
type HistoricSqlRedactionPattern,
|
||||
} from './redaction.js';
|
||||
import { includedQueryHistoryTableRefs } from './scope-membership.js';
|
||||
import {
|
||||
aggregatedTemplateSchema,
|
||||
historicSqlUnifiedPullConfigSchema,
|
||||
type AggregatedTemplate,
|
||||
type HistoricSqlDialect,
|
||||
type HistoricSqlReader,
|
||||
} from './types.js';
|
||||
|
||||
export interface QueryHistoryFilterProposal {
|
||||
excludedRoles: Array<{ role: string; reason: string; pattern: string }>;
|
||||
consideredRoleCount: number;
|
||||
skipped: { reason: 'no-llm' | 'no-daemon' | 'no-in-scope-history' | 'user-block-present' } | null;
|
||||
warnings: string[];
|
||||
parseFailedTemplateIds: string[];
|
||||
}
|
||||
|
||||
export interface ProposeQueryHistoryServiceAccountFiltersInput {
|
||||
connectionId: string;
|
||||
dialect: HistoricSqlDialect;
|
||||
queryClient: unknown;
|
||||
reader: HistoricSqlReader;
|
||||
sqlAnalysis: SqlAnalysisPort;
|
||||
llmRuntime: KtxLlmRuntimePort | null;
|
||||
pullConfig: unknown;
|
||||
now?: Date;
|
||||
userServiceAccountsPresent?: boolean;
|
||||
}
|
||||
|
||||
interface ParsedTemplateForPicker {
|
||||
template: AggregatedTemplate;
|
||||
tablesTouched: KtxTableRef[];
|
||||
includedTables: KtxTableRef[];
|
||||
}
|
||||
|
||||
interface RoleAccumulator {
|
||||
role: string;
|
||||
executions: number;
|
||||
distinctUsers: number;
|
||||
lastSeen: string;
|
||||
tables: Map<string, KtxTableRef>;
|
||||
templates: AggregatedTemplate[];
|
||||
}
|
||||
|
||||
interface QueryHistoryRoleRecord {
|
||||
role: string;
|
||||
inScopeTables: string[];
|
||||
executionsBucket: string;
|
||||
distinctUsersBucket: string;
|
||||
recencyBucket: string;
|
||||
representativeTemplates: Array<{ id: string; canonicalSql: string; dialect: HistoricSqlDialect }>;
|
||||
}
|
||||
|
||||
const queryHistoryFilterAdjudicationSchema = z.object({
|
||||
roles: z.array(
|
||||
z.object({
|
||||
role: z.string().min(1),
|
||||
exclude: z.boolean(),
|
||||
reason: z.string().min(1),
|
||||
}).strict(),
|
||||
),
|
||||
}).strict();
|
||||
|
||||
type QueryHistoryFilterAdjudication = z.infer<typeof queryHistoryFilterAdjudicationSchema>;
|
||||
|
||||
function emptyProposal(skipped: QueryHistoryFilterProposal['skipped'], warnings: string[] = []): QueryHistoryFilterProposal {
|
||||
return { excludedRoles: [], consideredRoleCount: 0, skipped, warnings, parseFailedTemplateIds: [] };
|
||||
}
|
||||
|
||||
function displayTableRef(ref: KtxTableRef): string {
|
||||
return [ref.catalog, ref.db, ref.name].filter((part): part is string => !!part && part.length > 0).join('.');
|
||||
}
|
||||
|
||||
function redactTemplateSqlForPicker(
|
||||
template: AggregatedTemplate,
|
||||
redactors: readonly HistoricSqlRedactionPattern[],
|
||||
): AggregatedTemplate {
|
||||
if (redactors.length === 0) {
|
||||
return template;
|
||||
}
|
||||
return {
|
||||
...template,
|
||||
canonicalSql: redactHistoricSqlText(template.canonicalSql, redactors),
|
||||
};
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function regexEscapeForExactRolePattern(role: string): string {
|
||||
return `^${role.replace(/[\\^$.*+?()[\]{}|]/g, '\\$&')}$`;
|
||||
}
|
||||
|
||||
function recordRole(
|
||||
acc: RoleAccumulator,
|
||||
template: AggregatedTemplate,
|
||||
tables: readonly KtxTableRef[],
|
||||
executions: number,
|
||||
): void {
|
||||
acc.executions += executions;
|
||||
acc.distinctUsers = Math.max(acc.distinctUsers, template.stats.distinctUsers);
|
||||
acc.lastSeen = template.stats.lastSeen > acc.lastSeen ? template.stats.lastSeen : acc.lastSeen;
|
||||
for (const table of tables) {
|
||||
acc.tables.set(tableRefKey(table), table);
|
||||
}
|
||||
acc.templates.push(template);
|
||||
}
|
||||
|
||||
function roleRecords(parsedTemplates: readonly ParsedTemplateForPicker[], now: Date): QueryHistoryRoleRecord[] {
|
||||
const byRole = new Map<string, RoleAccumulator>();
|
||||
for (const parsed of parsedTemplates) {
|
||||
for (const entry of parsed.template.topUsers) {
|
||||
if (!entry.user || entry.user.trim().length === 0 || entry.executions <= 0) {
|
||||
continue;
|
||||
}
|
||||
const role = entry.user.trim();
|
||||
const acc =
|
||||
byRole.get(role) ??
|
||||
{
|
||||
role,
|
||||
executions: 0,
|
||||
distinctUsers: 0,
|
||||
lastSeen: '1970-01-01T00:00:00.000Z',
|
||||
tables: new Map<string, KtxTableRef>(),
|
||||
templates: [],
|
||||
};
|
||||
recordRole(acc, parsed.template, parsed.includedTables, entry.executions);
|
||||
byRole.set(role, acc);
|
||||
}
|
||||
}
|
||||
|
||||
return [...byRole.values()]
|
||||
.sort((left, right) => right.executions - left.executions || left.role.localeCompare(right.role))
|
||||
.map((acc) => ({
|
||||
role: acc.role,
|
||||
inScopeTables: [...acc.tables.entries()]
|
||||
.sort(([left], [right]) => left.localeCompare(right))
|
||||
.slice(0, 25)
|
||||
.map(([, ref]) => displayTableRef(ref)),
|
||||
executionsBucket: bucketExecutions(acc.executions),
|
||||
distinctUsersBucket: bucketDistinctUsers(acc.distinctUsers),
|
||||
recencyBucket: bucketRecency(acc.lastSeen, now),
|
||||
representativeTemplates: [...acc.templates]
|
||||
.sort((left, right) => right.stats.executions - left.stats.executions || left.templateId.localeCompare(right.templateId))
|
||||
.slice(0, 3)
|
||||
.map((template) => ({
|
||||
id: template.templateId,
|
||||
canonicalSql: template.canonicalSql,
|
||||
dialect: template.dialect,
|
||||
})),
|
||||
}));
|
||||
}
|
||||
|
||||
function adjudicationSystemPrompt(): string {
|
||||
return [
|
||||
'You are helping ktx decide whether observed query-history roles are operational service accounts.',
|
||||
'Default every role to keep. Mark exclude true only when the aggregate evidence clearly shows loader, ELT, reverse-ETL, export, refresh, or maintenance traffic rather than analyst or BI-dashboard usage.',
|
||||
'Use only the observed role records. Do not rely on a hardcoded denylist. Return structured output only.',
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
export async function proposeQueryHistoryServiceAccountFilters(
|
||||
input: ProposeQueryHistoryServiceAccountFiltersInput,
|
||||
): Promise<QueryHistoryFilterProposal> {
|
||||
if (!input.llmRuntime) {
|
||||
return emptyProposal({ reason: 'no-llm' });
|
||||
}
|
||||
|
||||
const config = historicSqlUnifiedPullConfigSchema.parse(input.pullConfig);
|
||||
const redactors = compileHistoricSqlRedactionPatterns(config.redactionPatterns);
|
||||
const now = input.now ?? new Date();
|
||||
const windowDays = 'windowDays' in config ? config.windowDays : 90;
|
||||
const windowStart = new Date(now.getTime() - windowDays * 24 * 60 * 60 * 1000);
|
||||
const warnings: string[] = [];
|
||||
const parseFailedTemplateIds: string[] = [];
|
||||
const snapshot: AggregatedTemplate[] = [];
|
||||
|
||||
try {
|
||||
for await (const row of input.reader.fetchAggregated(input.queryClient, { start: windowStart, end: now }, config)) {
|
||||
snapshot.push(aggregatedTemplateSchema.parse(row));
|
||||
}
|
||||
} catch (error) {
|
||||
return emptyProposal(null, [
|
||||
`query_history_filter_picker_read_failed:${error instanceof Error ? error.message : String(error)}`,
|
||||
]);
|
||||
}
|
||||
|
||||
if (snapshot.length === 0) {
|
||||
return emptyProposal({ reason: 'no-in-scope-history' });
|
||||
}
|
||||
|
||||
const analysisItems = snapshot.map((template) => ({ id: template.templateId, sql: template.canonicalSql }));
|
||||
const analysisOptions =
|
||||
config.modeledTableCatalog.length > 0 ? { catalog: { tables: config.modeledTableCatalog } } : undefined;
|
||||
let analysis: Awaited<ReturnType<SqlAnalysisPort['analyzeBatch']>>;
|
||||
try {
|
||||
analysis = await input.sqlAnalysis.analyzeBatch(analysisItems, input.dialect, analysisOptions);
|
||||
} catch (error) {
|
||||
return emptyProposal({ reason: 'no-daemon' }, [
|
||||
`query_history_filter_picker_analysis_failed:${error instanceof Error ? error.message : String(error)}`,
|
||||
]);
|
||||
}
|
||||
|
||||
const parsedTemplates: ParsedTemplateForPicker[] = [];
|
||||
for (const template of snapshot) {
|
||||
const parsed = analysis.get(template.templateId);
|
||||
if (!parsed || parsed.error) {
|
||||
parseFailedTemplateIds.push(template.templateId);
|
||||
continue;
|
||||
}
|
||||
const tablesTouched = [...new Map(parsed.tablesTouched.map((ref) => [tableRefKey(ref), ref])).values()]
|
||||
.filter((ref) => ref.name.length > 0)
|
||||
.sort((left, right) => tableRefKey(left).localeCompare(tableRefKey(right)));
|
||||
const includedTables = includedQueryHistoryTableRefs(tablesTouched, config);
|
||||
if (includedTables.length === 0) {
|
||||
continue;
|
||||
}
|
||||
parsedTemplates.push({
|
||||
template: redactTemplateSqlForPicker(template, redactors),
|
||||
tablesTouched,
|
||||
includedTables,
|
||||
});
|
||||
}
|
||||
|
||||
const records = roleRecords(parsedTemplates, now);
|
||||
if (records.length <= 1) {
|
||||
return {
|
||||
excludedRoles: [],
|
||||
consideredRoleCount: records.length,
|
||||
skipped: { reason: 'no-in-scope-history' },
|
||||
warnings,
|
||||
parseFailedTemplateIds,
|
||||
};
|
||||
}
|
||||
|
||||
let generated: QueryHistoryFilterAdjudication;
|
||||
try {
|
||||
generated = await input.llmRuntime.generateObject<QueryHistoryFilterAdjudication, typeof queryHistoryFilterAdjudicationSchema>({
|
||||
role: 'candidateExtraction',
|
||||
system: adjudicationSystemPrompt(),
|
||||
prompt: JSON.stringify({ connectionId: input.connectionId, dialect: input.dialect, roles: records }),
|
||||
schema: queryHistoryFilterAdjudicationSchema,
|
||||
});
|
||||
} catch (error) {
|
||||
return {
|
||||
excludedRoles: [],
|
||||
consideredRoleCount: records.length,
|
||||
skipped: { reason: 'no-llm' },
|
||||
warnings: [
|
||||
...warnings,
|
||||
`query_history_filter_picker_llm_failed:${error instanceof Error ? error.message : String(error)}`,
|
||||
],
|
||||
parseFailedTemplateIds,
|
||||
};
|
||||
}
|
||||
|
||||
const knownRoles = new Set(records.map((record) => record.role));
|
||||
const excludedRoles = generated.roles
|
||||
.filter((role) => role.exclude && knownRoles.has(role.role))
|
||||
.sort((left, right) => left.role.localeCompare(right.role))
|
||||
.map((role) => ({
|
||||
role: role.role,
|
||||
reason: role.reason,
|
||||
pattern: regexEscapeForExactRolePattern(role.role),
|
||||
}));
|
||||
|
||||
return {
|
||||
excludedRoles,
|
||||
consideredRoleCount: records.length,
|
||||
skipped: input.userServiceAccountsPresent ? { reason: 'user-block-present' } : null,
|
||||
warnings,
|
||||
parseFailedTemplateIds,
|
||||
};
|
||||
}
|
||||
|
|
@ -1,260 +0,0 @@
|
|||
import type { Dirent } from 'node:fs';
|
||||
import { access, readdir, readFile } from 'node:fs/promises';
|
||||
import { join, relative } from 'node:path';
|
||||
import YAML from 'yaml';
|
||||
import { getDriverRegistration } from '../../../connections/drivers.js';
|
||||
import { parseDottedTableEntry } from '../../../scan/enabled-tables.js';
|
||||
import { tableRefKey, tableRefSet, type KtxTableRefKey } from '../../../scan/table-ref.js';
|
||||
import type { KtxTableRef } from '../../../scan/types.js';
|
||||
import { readLiveDatabaseTableFiles } from '../live-database/stage.js';
|
||||
|
||||
export interface QueryHistoryScopeFloorInput {
|
||||
projectDir: string;
|
||||
connectionId: string;
|
||||
driver: string;
|
||||
connection: Record<string, unknown>;
|
||||
storedQueryHistory: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface QueryHistoryScopeFloor {
|
||||
enabledTables: KtxTableRef[];
|
||||
enabledTableKeys: ReadonlySet<KtxTableRefKey> | null;
|
||||
enabledSchemas: string[];
|
||||
modeledTableCatalog: KtxTableRef[];
|
||||
floorDisabled: boolean;
|
||||
warnings: string[];
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function stringArray(value: unknown): string[] {
|
||||
return Array.isArray(value)
|
||||
? value
|
||||
.filter((item): item is string => typeof item === 'string' && item.trim().length > 0)
|
||||
.map((item) => item.trim())
|
||||
: [];
|
||||
}
|
||||
|
||||
function tableRefsFromValues(values: unknown): KtxTableRef[] {
|
||||
if (!Array.isArray(values)) return [];
|
||||
return values.flatMap((value) => {
|
||||
if (typeof value === 'string') {
|
||||
const ref = parseDottedTableEntry(value);
|
||||
return ref ? [ref] : [];
|
||||
}
|
||||
if (isRecord(value) && typeof value.name === 'string' && value.name.length > 0) {
|
||||
return [
|
||||
{
|
||||
catalog: typeof value.catalog === 'string' ? value.catalog : null,
|
||||
db: typeof value.db === 'string' ? value.db : null,
|
||||
name: value.name,
|
||||
},
|
||||
];
|
||||
}
|
||||
return [];
|
||||
});
|
||||
}
|
||||
|
||||
function declaredSchemas(driver: string, connection: Record<string, unknown>): string[] {
|
||||
const key = getDriverRegistration(driver)?.scopeConfigKey;
|
||||
if (!key) return [];
|
||||
return [...new Set(stringArray(connection[key]))].sort();
|
||||
}
|
||||
|
||||
function uniqueSortedTableRefs(refs: readonly KtxTableRef[]): KtxTableRef[] {
|
||||
const byKey = new Map<KtxTableRefKey, KtxTableRef>();
|
||||
for (const ref of refs) {
|
||||
byKey.set(tableRefKey(ref), ref);
|
||||
}
|
||||
return [...byKey.entries()]
|
||||
.sort(([left], [right]) => left.localeCompare(right))
|
||||
.map(([, ref]) => ref);
|
||||
}
|
||||
|
||||
async function latestLiveDatabaseScanDir(projectDir: string, connectionId: string): Promise<string | null> {
|
||||
const root = join(projectDir, 'raw-sources', connectionId, 'live-database');
|
||||
let entries: Dirent[];
|
||||
try {
|
||||
entries = await readdir(root, { withFileTypes: true });
|
||||
} catch (error) {
|
||||
if (error && typeof error === 'object' && 'code' in error && error.code === 'ENOENT') return null;
|
||||
throw error;
|
||||
}
|
||||
const syncDirs = entries
|
||||
.filter((entry) => entry.isDirectory())
|
||||
.map((entry) => entry.name)
|
||||
.sort()
|
||||
.reverse();
|
||||
for (const syncDir of syncDirs) {
|
||||
const absolute = join(root, syncDir);
|
||||
try {
|
||||
await access(join(absolute, 'connection.json'));
|
||||
return absolute;
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function scannedTableRefs(
|
||||
projectDir: string,
|
||||
connectionId: string,
|
||||
): Promise<{ refs: KtxTableRef[]; catalogAvailable: boolean; warnings: string[] }> {
|
||||
const scanDir = await latestLiveDatabaseScanDir(projectDir, connectionId);
|
||||
if (!scanDir) {
|
||||
return { refs: [], catalogAvailable: false, warnings: [] };
|
||||
}
|
||||
try {
|
||||
const tableFiles = await readLiveDatabaseTableFiles(scanDir);
|
||||
return {
|
||||
refs: uniqueSortedTableRefs(
|
||||
tableFiles.map(({ table }) => ({ catalog: table.catalog, db: table.db, name: table.name })),
|
||||
),
|
||||
catalogAvailable: true,
|
||||
warnings: [],
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
refs: [],
|
||||
catalogAvailable: false,
|
||||
warnings: [
|
||||
`query_history_scope_floor_catalog_read_failed:live_database_scan:${error instanceof Error ? error.message : String(error)}`,
|
||||
],
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function listYamlFiles(root: string): Promise<string[]> {
|
||||
try {
|
||||
const entries = await readdir(root, { withFileTypes: true, recursive: true });
|
||||
return entries
|
||||
.filter((entry) => entry.isFile() && /\.ya?ml$/i.test(entry.name))
|
||||
.map((entry) => relative(root, join(entry.parentPath, entry.name)).replace(/\\/g, '/'))
|
||||
.sort();
|
||||
} catch (error) {
|
||||
if (error && typeof error === 'object' && 'code' in error && error.code === 'ENOENT') return [];
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function refsFromManifest(content: string): KtxTableRef[] {
|
||||
const parsed = YAML.parse(content) as unknown;
|
||||
if (!isRecord(parsed) || !isRecord(parsed.tables)) return [];
|
||||
return Object.values(parsed.tables).flatMap((entry) => {
|
||||
if (!isRecord(entry) || typeof entry.table !== 'string') return [];
|
||||
const ref = parseDottedTableEntry(entry.table);
|
||||
return ref ? [ref] : [];
|
||||
});
|
||||
}
|
||||
|
||||
function refsFromStandaloneSource(content: string): KtxTableRef[] {
|
||||
const parsed = YAML.parse(content) as unknown;
|
||||
if (!isRecord(parsed) || typeof parsed.table !== 'string') return [];
|
||||
const ref = parseDottedTableEntry(parsed.table);
|
||||
return ref ? [ref] : [];
|
||||
}
|
||||
|
||||
async function semanticTableRefs(
|
||||
projectDir: string,
|
||||
connectionId: string,
|
||||
): Promise<{ refs: KtxTableRef[]; warnings: string[] }> {
|
||||
const root = join(projectDir, 'semantic-layer', connectionId);
|
||||
const files = await listYamlFiles(root);
|
||||
const refs: KtxTableRef[] = [];
|
||||
const warnings: string[] = [];
|
||||
for (const file of files) {
|
||||
try {
|
||||
const content = await readFile(join(root, file), 'utf-8');
|
||||
refs.push(...(file.startsWith('_schema/') ? refsFromManifest(content) : refsFromStandaloneSource(content)));
|
||||
} catch (error) {
|
||||
warnings.push(
|
||||
`query_history_scope_floor_catalog_read_failed:${file}:${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
return { refs: uniqueSortedTableRefs(refs), warnings };
|
||||
}
|
||||
|
||||
export async function resolveQueryHistoryScopeFloor(input: QueryHistoryScopeFloorInput): Promise<QueryHistoryScopeFloor> {
|
||||
const explicitEnabledTables = [
|
||||
...tableRefsFromValues(input.storedQueryHistory.enabledTables),
|
||||
...tableRefsFromValues(input.connection.enabled_tables),
|
||||
];
|
||||
const semanticTables = await semanticTableRefs(input.projectDir, input.connectionId);
|
||||
const scannedTables = await scannedTableRefs(input.projectDir, input.connectionId);
|
||||
const modeledTables = uniqueSortedTableRefs([
|
||||
...semanticTables.refs,
|
||||
...scannedTables.refs,
|
||||
...explicitEnabledTables,
|
||||
]);
|
||||
const warnings = [...semanticTables.warnings, ...scannedTables.warnings];
|
||||
|
||||
if (explicitEnabledTables.length > 0) {
|
||||
return {
|
||||
enabledTables: explicitEnabledTables,
|
||||
enabledTableKeys: tableRefSet(explicitEnabledTables),
|
||||
enabledSchemas: [],
|
||||
modeledTableCatalog: modeledTables,
|
||||
floorDisabled: false,
|
||||
warnings,
|
||||
};
|
||||
}
|
||||
|
||||
const explicitSchemas = stringArray(input.storedQueryHistory.enabledSchemas);
|
||||
if (explicitSchemas.includes('*')) {
|
||||
return {
|
||||
enabledTables: [],
|
||||
enabledTableKeys: null,
|
||||
enabledSchemas: ['*'],
|
||||
modeledTableCatalog: modeledTables,
|
||||
floorDisabled: true,
|
||||
warnings,
|
||||
};
|
||||
}
|
||||
if (explicitSchemas.length > 0) {
|
||||
if (!scannedTables.catalogAvailable || modeledTables.length === 0) {
|
||||
return {
|
||||
enabledTables: [],
|
||||
enabledTableKeys: null,
|
||||
enabledSchemas: ['*'],
|
||||
modeledTableCatalog: modeledTables,
|
||||
floorDisabled: true,
|
||||
warnings: [...warnings, 'query_history_scope_floor_disabled:catalog_unavailable'],
|
||||
};
|
||||
}
|
||||
return {
|
||||
enabledTables: [],
|
||||
enabledTableKeys: null,
|
||||
enabledSchemas: [...new Set(explicitSchemas)].sort(),
|
||||
modeledTableCatalog: modeledTables,
|
||||
floorDisabled: false,
|
||||
warnings,
|
||||
};
|
||||
}
|
||||
|
||||
const schemas = new Set(declaredSchemas(input.driver, input.connection));
|
||||
for (const ref of semanticTables.refs) {
|
||||
if (ref.db) schemas.add(ref.db);
|
||||
}
|
||||
if (schemas.size > 0 && (!scannedTables.catalogAvailable || modeledTables.length === 0)) {
|
||||
return {
|
||||
enabledTables: [],
|
||||
enabledTableKeys: null,
|
||||
enabledSchemas: ['*'],
|
||||
modeledTableCatalog: modeledTables,
|
||||
floorDisabled: true,
|
||||
warnings: [...warnings, 'query_history_scope_floor_disabled:catalog_unavailable'],
|
||||
};
|
||||
}
|
||||
return {
|
||||
enabledTables: [],
|
||||
enabledTableKeys: null,
|
||||
enabledSchemas: [...schemas].sort(),
|
||||
modeledTableCatalog: modeledTables,
|
||||
floorDisabled: false,
|
||||
warnings,
|
||||
};
|
||||
}
|
||||
|
|
@ -1,45 +0,0 @@
|
|||
import { tableRefKey, tableRefSet } from '../../../scan/table-ref.js';
|
||||
import type { KtxTableRef } from '../../../scan/types.js';
|
||||
|
||||
export interface QueryHistoryScopeMembershipConfig {
|
||||
enabledTables: readonly KtxTableRef[];
|
||||
enabledSchemas: readonly string[];
|
||||
}
|
||||
|
||||
function schemaNameForRef(ref: KtxTableRef): string | null {
|
||||
return ref.db && ref.db.length > 0 ? ref.db : null;
|
||||
}
|
||||
|
||||
function schemaNamesFromConfig(enabledSchemas: readonly string[]): Set<string> {
|
||||
return new Set(enabledSchemas.filter((schema) => schema !== '*'));
|
||||
}
|
||||
|
||||
export function isQueryHistoryScopeFloorDisabled(config: QueryHistoryScopeMembershipConfig): boolean {
|
||||
return config.enabledSchemas.includes('*');
|
||||
}
|
||||
|
||||
export function shouldFailOpenQueryHistoryScope(config: QueryHistoryScopeMembershipConfig): boolean {
|
||||
return (
|
||||
config.enabledTables.length === 0 &&
|
||||
!isQueryHistoryScopeFloorDisabled(config) &&
|
||||
config.enabledSchemas.length === 0
|
||||
);
|
||||
}
|
||||
|
||||
export function includedQueryHistoryTableRefs(
|
||||
tablesTouched: readonly KtxTableRef[],
|
||||
config: QueryHistoryScopeMembershipConfig,
|
||||
): KtxTableRef[] {
|
||||
if (config.enabledTables.length > 0) {
|
||||
const enabled = tableRefSet(config.enabledTables);
|
||||
return tablesTouched.filter((ref) => enabled.has(tableRefKey(ref)));
|
||||
}
|
||||
if (isQueryHistoryScopeFloorDisabled(config) || shouldFailOpenQueryHistoryScope(config)) {
|
||||
return [...tablesTouched];
|
||||
}
|
||||
const schemas = schemaNamesFromConfig(config.enabledSchemas);
|
||||
return tablesTouched.filter((ref) => {
|
||||
const schema = schemaNameForRef(ref);
|
||||
return schema !== null && schemas.has(schema);
|
||||
});
|
||||
}
|
||||
|
|
@ -188,75 +188,26 @@ export class SnowflakeHistoricSqlQueryHistoryReader {
|
|||
config: HistoricSqlUnifiedPullConfig,
|
||||
): AsyncIterable<AggregatedTemplate> {
|
||||
const sql = `
|
||||
WITH filtered_queries AS (
|
||||
SELECT
|
||||
query_hash,
|
||||
query_text,
|
||||
user_name,
|
||||
start_time,
|
||||
total_elapsed_time,
|
||||
execution_status,
|
||||
rows_produced
|
||||
FROM SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY
|
||||
WHERE query_text IS NOT NULL
|
||||
AND query_type IN ('SELECT', 'MERGE')
|
||||
AND start_time >= ${timestampLiteral(window.start)}
|
||||
AND start_time < ${timestampLiteral(window.end)}
|
||||
),
|
||||
template_stats AS (
|
||||
SELECT
|
||||
query_hash AS template_id,
|
||||
MIN(query_text) AS canonical_sql,
|
||||
COUNT(*) AS executions,
|
||||
COUNT(DISTINCT user_name) AS distinct_users,
|
||||
MIN(start_time) AS first_seen,
|
||||
MAX(start_time) AS last_seen,
|
||||
APPROX_PERCENTILE(total_elapsed_time, 0.50) AS p50_ms,
|
||||
APPROX_PERCENTILE(total_elapsed_time, 0.95) AS p95_ms,
|
||||
DIV0(COUNT_IF(execution_status != 'SUCCESS'), COUNT(*)) AS error_rate,
|
||||
SUM(rows_produced) AS rows_produced
|
||||
FROM filtered_queries
|
||||
GROUP BY query_hash
|
||||
HAVING COUNT(*) >= ${config.minExecutions}
|
||||
),
|
||||
template_users AS (
|
||||
SELECT
|
||||
query_hash AS template_id,
|
||||
user_name AS user,
|
||||
COUNT(*) AS executions,
|
||||
MAX(start_time) AS last_seen
|
||||
FROM filtered_queries
|
||||
GROUP BY query_hash, user_name
|
||||
)
|
||||
SELECT
|
||||
stats.template_id,
|
||||
stats.canonical_sql,
|
||||
stats.executions,
|
||||
stats.distinct_users,
|
||||
stats.first_seen,
|
||||
stats.last_seen,
|
||||
stats.p50_ms,
|
||||
stats.p95_ms,
|
||||
stats.error_rate,
|
||||
stats.rows_produced,
|
||||
ARRAY_AGG(
|
||||
OBJECT_CONSTRUCT('user', users.user, 'executions', users.executions)
|
||||
) WITHIN GROUP (ORDER BY users.executions DESC, users.last_seen DESC)::string AS top_users
|
||||
FROM template_stats AS stats
|
||||
JOIN template_users AS users
|
||||
ON users.template_id = stats.template_id
|
||||
GROUP BY
|
||||
stats.template_id,
|
||||
stats.canonical_sql,
|
||||
stats.executions,
|
||||
stats.distinct_users,
|
||||
stats.first_seen,
|
||||
stats.last_seen,
|
||||
stats.p50_ms,
|
||||
stats.p95_ms,
|
||||
stats.error_rate,
|
||||
stats.rows_produced
|
||||
ORDER BY stats.executions DESC`.trim();
|
||||
query_hash AS template_id,
|
||||
MIN(query_text) AS canonical_sql,
|
||||
COUNT(*) AS executions,
|
||||
COUNT(DISTINCT user_name) AS distinct_users,
|
||||
MIN(start_time) AS first_seen,
|
||||
MAX(start_time) AS last_seen,
|
||||
APPROX_PERCENTILE(total_elapsed_time, 0.50) AS p50_ms,
|
||||
APPROX_PERCENTILE(total_elapsed_time, 0.95) AS p95_ms,
|
||||
DIV0(COUNT_IF(execution_status != 'SUCCESS'), COUNT(*)) AS error_rate,
|
||||
SUM(rows_produced) AS rows_produced,
|
||||
ARRAY_AGG(OBJECT_CONSTRUCT('user', user_name, 'executions', 1)) WITHIN GROUP (ORDER BY start_time DESC)::string AS top_users
|
||||
FROM SNOWFLAKE.ACCOUNT_USAGE.QUERY_HISTORY
|
||||
WHERE query_text IS NOT NULL
|
||||
AND query_type IN ('SELECT', 'MERGE')
|
||||
AND start_time >= ${timestampLiteral(window.start)}
|
||||
AND start_time < ${timestampLiteral(window.end)}
|
||||
GROUP BY query_hash
|
||||
HAVING COUNT(*) >= ${config.minExecutions}
|
||||
ORDER BY executions DESC`.trim();
|
||||
const result = await queryClient(client).executeQuery(sql);
|
||||
if (result.error) {
|
||||
throw grantsError(result.error);
|
||||
|
|
|
|||
|
|
@ -1,8 +1,6 @@
|
|||
import { mkdir, writeFile } from 'node:fs/promises';
|
||||
import { dirname, join } from 'node:path';
|
||||
import type { SqlAnalysisPort } from '../../../../context/sql-analysis/ports.js';
|
||||
import { tableRefKey, type KtxTableRefKey } from '../../../scan/table-ref.js';
|
||||
import type { KtxTableRef } from '../../../scan/types.js';
|
||||
import {
|
||||
bucketDistinctUsers,
|
||||
bucketErrorRate,
|
||||
|
|
@ -17,11 +15,6 @@ import {
|
|||
redactHistoricSqlText,
|
||||
type HistoricSqlRedactionPattern,
|
||||
} from './redaction.js';
|
||||
import {
|
||||
includedQueryHistoryTableRefs,
|
||||
isQueryHistoryScopeFloorDisabled,
|
||||
shouldFailOpenQueryHistoryScope,
|
||||
} from './scope-membership.js';
|
||||
import {
|
||||
HISTORIC_SQL_SOURCE_KEY,
|
||||
aggregatedTemplateSchema,
|
||||
|
|
@ -45,13 +38,17 @@ interface StageHistoricSqlAggregatedSnapshotInput {
|
|||
|
||||
interface ParsedTemplate {
|
||||
template: AggregatedTemplate;
|
||||
tablesTouched: KtxTableRef[];
|
||||
includedTables: KtxTableRef[];
|
||||
tablesTouched: string[];
|
||||
includedTables: string[];
|
||||
columnsByClause: Record<string, string[]>;
|
||||
}
|
||||
|
||||
interface EnabledTableFilter {
|
||||
exact: Set<string>;
|
||||
uniqueUnqualified: Set<string>;
|
||||
}
|
||||
|
||||
interface TableAccumulator {
|
||||
tableRef: KtxTableRef;
|
||||
table: string;
|
||||
executions: number;
|
||||
distinctUsers: number;
|
||||
|
|
@ -82,21 +79,8 @@ function matchesAny(value: string | null, patterns: RegExp[]): boolean {
|
|||
return !!value && patterns.some((pattern) => pattern.test(value));
|
||||
}
|
||||
|
||||
// ktx's own warehouse scan emits relationship- and column-profiling probes that land in
|
||||
// pg_stat_statements (relationship-validation, relationship-composite-candidates, and each
|
||||
// dialect's relationship value aggregation). They are ktx introspection, not genuine query
|
||||
// usage, so they must not be mined back as query history. The markers are ktx-owned
|
||||
// identifiers, stable across dialects.
|
||||
function isKtxScanProbe(sql: string): boolean {
|
||||
if (/\brelationship_profile_values\b/i.test(sql)) {
|
||||
return true;
|
||||
}
|
||||
return /\bchild_values\b/i.test(sql) && /\bparent_values\b/i.test(sql);
|
||||
}
|
||||
|
||||
function shouldDropBySql(sql: string, config: HistoricSqlUnifiedPullConfig): boolean {
|
||||
if (NOISE_PREFIX_RE.test(sql) || SYSTEM_TABLE_RE.test(sql)) return true;
|
||||
if (isKtxScanProbe(sql)) return true;
|
||||
if (config.filters.dropTrivialProbes !== false && TRIVIAL_SQL_RE.test(sql)) return true;
|
||||
return false;
|
||||
}
|
||||
|
|
@ -108,7 +92,8 @@ function shouldDropByUsers(template: AggregatedTemplate, config: HistoricSqlUnif
|
|||
const matchingExecutions = template.topUsers
|
||||
.filter((entry) => matchesAny(entry.user, patterns))
|
||||
.reduce((sum, entry) => sum + entry.executions, 0);
|
||||
const serviceOnly = template.stats.executions > 0 && matchingExecutions >= template.stats.executions;
|
||||
const allExecutions = template.topUsers.reduce((sum, entry) => sum + entry.executions, 0);
|
||||
const serviceOnly = allExecutions > 0 && matchingExecutions >= allExecutions;
|
||||
return service.mode === 'exclude' ? serviceOnly : !serviceOnly;
|
||||
}
|
||||
|
||||
|
|
@ -124,8 +109,43 @@ function shouldDropTemplate(template: AggregatedTemplate, config: HistoricSqlUni
|
|||
return false;
|
||||
}
|
||||
|
||||
function displayTableRef(ref: KtxTableRef): string {
|
||||
return [ref.catalog, ref.db, ref.name].filter((part): part is string => !!part && part.length > 0).join('.');
|
||||
function normalizeTableIdentifier(value: string): string {
|
||||
return value.trim().toLowerCase();
|
||||
}
|
||||
|
||||
function unqualifiedTableIdentifier(value: string): string {
|
||||
const parts = normalizeTableIdentifier(value).split('.').filter(Boolean);
|
||||
return parts.at(-1) ?? '';
|
||||
}
|
||||
|
||||
function buildEnabledTableFilter(enabledTables: string[]): EnabledTableFilter | null {
|
||||
if (enabledTables.length === 0) {
|
||||
return null;
|
||||
}
|
||||
const exact = new Set(enabledTables.map(normalizeTableIdentifier).filter((value) => value.length > 0));
|
||||
const unqualifiedCounts = new Map<string, number>();
|
||||
for (const table of exact) {
|
||||
const unqualified = unqualifiedTableIdentifier(table);
|
||||
if (unqualified.length > 0) {
|
||||
unqualifiedCounts.set(unqualified, (unqualifiedCounts.get(unqualified) ?? 0) + 1);
|
||||
}
|
||||
}
|
||||
return {
|
||||
exact,
|
||||
uniqueUnqualified: new Set(
|
||||
[...unqualifiedCounts.entries()]
|
||||
.filter(([, count]) => count === 1)
|
||||
.map(([table]) => table),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
function isEnabledTable(table: string, filter: EnabledTableFilter | null): boolean {
|
||||
if (!filter) {
|
||||
return true;
|
||||
}
|
||||
const normalized = normalizeTableIdentifier(table);
|
||||
return filter.exact.has(normalized) || filter.uniqueUnqualified.has(unqualifiedTableIdentifier(normalized));
|
||||
}
|
||||
|
||||
function historicSqlWindowDays(config: HistoricSqlUnifiedPullConfig): number {
|
||||
|
|
@ -160,10 +180,9 @@ function recordJoin(acc: TableAccumulator, otherTable: string, columns: string[]
|
|||
}
|
||||
}
|
||||
|
||||
function accumulatorFor(tableRef: KtxTableRef): TableAccumulator {
|
||||
function accumulatorFor(table: string): TableAccumulator {
|
||||
return {
|
||||
tableRef,
|
||||
table: displayTableRef(tableRef),
|
||||
table,
|
||||
executions: 0,
|
||||
distinctUsers: 0,
|
||||
errorRateNumerator: 0,
|
||||
|
|
@ -193,8 +212,8 @@ function addTemplate(acc: TableAccumulator, parsed: ParsedTemplate): void {
|
|||
}
|
||||
}
|
||||
const joinColumns = parsed.columnsByClause.join ?? [];
|
||||
for (const otherTable of parsed.tablesTouched.filter((table) => tableRefKey(table) !== tableRefKey(acc.tableRef))) {
|
||||
recordJoin(acc, displayTableRef(otherTable), joinColumns, executions);
|
||||
for (const otherTable of parsed.tablesTouched.filter((table) => table !== acc.table)) {
|
||||
recordJoin(acc, otherTable, joinColumns, executions);
|
||||
}
|
||||
acc.topTemplates.push(parsed.template);
|
||||
}
|
||||
|
|
@ -231,7 +250,6 @@ function toStagedTable(acc: TableAccumulator, now: Date): StagedTableInput {
|
|||
|
||||
return {
|
||||
table: acc.table,
|
||||
tableRef: acc.tableRef,
|
||||
stats: {
|
||||
executionsBucket: bucketExecutions(acc.executions),
|
||||
distinctUsersBucket: bucketDistinctUsers(acc.distinctUsers),
|
||||
|
|
@ -251,7 +269,7 @@ function toPatternsInput(parsedTemplates: ParsedTemplate[]): StagedPatternsInput
|
|||
.map(({ template, tablesTouched }) => ({
|
||||
id: template.templateId,
|
||||
canonicalSql: template.canonicalSql,
|
||||
tablesTouched: [...tablesTouched].sort((left, right) => tableRefKey(left).localeCompare(tableRefKey(right))),
|
||||
tablesTouched: [...tablesTouched].sort(),
|
||||
executionsBucket: bucketExecutions(template.stats.executions),
|
||||
distinctUsersBucket: bucketDistinctUsers(template.stats.distinctUsers),
|
||||
dialect: template.dialect,
|
||||
|
|
@ -262,6 +280,7 @@ function toPatternsInput(parsedTemplates: ParsedTemplate[]): StagedPatternsInput
|
|||
|
||||
export async function stageHistoricSqlAggregatedSnapshot(input: StageHistoricSqlAggregatedSnapshotInput): Promise<void> {
|
||||
const config = historicSqlUnifiedPullConfigSchema.parse(input.pullConfig);
|
||||
const enabledTableFilter = buildEnabledTableFilter(config.enabledTables);
|
||||
const redactors = compileHistoricSqlRedactionPatterns(config.redactionPatterns);
|
||||
const now = input.now ?? new Date();
|
||||
const windowStart = new Date(now.getTime() - historicSqlWindowDays(config) * 24 * 60 * 60 * 1000);
|
||||
|
|
@ -277,25 +296,11 @@ export async function stageHistoricSqlAggregatedSnapshot(input: StageHistoricSql
|
|||
}
|
||||
}
|
||||
|
||||
const analysisItems = snapshot.map((template) => ({ id: template.templateId, sql: template.canonicalSql }));
|
||||
const analysisOptions =
|
||||
config.modeledTableCatalog.length > 0 ? { catalog: { tables: config.modeledTableCatalog } } : undefined;
|
||||
const warnings: string[] = [
|
||||
...config.scopeFloorWarnings,
|
||||
...(shouldFailOpenQueryHistoryScope(config) ? ['query_history_scope_floor_disabled:empty_modeled_scope'] : []),
|
||||
];
|
||||
let scopeDisabledByQualificationFailure = false;
|
||||
let analysis: Awaited<ReturnType<SqlAnalysisPort['analyzeBatch']>>;
|
||||
try {
|
||||
analysis = await input.sqlAnalysis.analyzeBatch(analysisItems, config.dialect, analysisOptions);
|
||||
} catch (error) {
|
||||
if (!analysisOptions || config.enabledTables.length > 0 || isQueryHistoryScopeFloorDisabled(config)) {
|
||||
throw error;
|
||||
}
|
||||
warnings.push('query_history_scope_floor_disabled:catalog_qualification_failed');
|
||||
scopeDisabledByQualificationFailure = true;
|
||||
analysis = await input.sqlAnalysis.analyzeBatch(analysisItems, config.dialect, undefined);
|
||||
}
|
||||
const analysis = await input.sqlAnalysis.analyzeBatch(
|
||||
snapshot.map((template) => ({ id: template.templateId, sql: template.canonicalSql })),
|
||||
config.dialect,
|
||||
);
|
||||
const warnings: string[] = [];
|
||||
const parsedTemplates: ParsedTemplate[] = [];
|
||||
for (const template of snapshot) {
|
||||
const parsed = analysis.get(template.templateId);
|
||||
|
|
@ -303,12 +308,8 @@ export async function stageHistoricSqlAggregatedSnapshot(input: StageHistoricSql
|
|||
warnings.push(`parse_failed:${template.templateId}`);
|
||||
continue;
|
||||
}
|
||||
const tablesTouched = [...new Map(parsed.tablesTouched.map((ref) => [tableRefKey(ref), ref])).values()]
|
||||
.filter((ref) => ref.name.length > 0)
|
||||
.sort((left, right) => tableRefKey(left).localeCompare(tableRefKey(right)));
|
||||
const includedTables = scopeDisabledByQualificationFailure
|
||||
? [...tablesTouched]
|
||||
: includedQueryHistoryTableRefs(tablesTouched, config);
|
||||
const tablesTouched = [...new Set(parsed.tablesTouched)].filter((table) => table.length > 0).sort();
|
||||
const includedTables = tablesTouched.filter((table) => isEnabledTable(table, enabledTableFilter));
|
||||
if (includedTables.length === 0) {
|
||||
continue;
|
||||
}
|
||||
|
|
@ -322,23 +323,22 @@ export async function stageHistoricSqlAggregatedSnapshot(input: StageHistoricSql
|
|||
});
|
||||
}
|
||||
|
||||
const byTable = new Map<KtxTableRefKey, TableAccumulator>();
|
||||
const byTable = new Map<string, TableAccumulator>();
|
||||
for (const parsed of parsedTemplates) {
|
||||
for (const tableRef of parsed.includedTables) {
|
||||
const key = tableRefKey(tableRef);
|
||||
const acc = byTable.get(key) ?? accumulatorFor(tableRef);
|
||||
for (const table of parsed.includedTables) {
|
||||
const acc = byTable.get(table) ?? accumulatorFor(table);
|
||||
addTemplate(acc, parsed);
|
||||
byTable.set(key, acc);
|
||||
byTable.set(table, acc);
|
||||
}
|
||||
}
|
||||
|
||||
await mkdir(input.stagedDir, { recursive: true });
|
||||
for (const [, acc] of [...byTable.entries()].sort((left, right) => left[0].localeCompare(right[0]))) {
|
||||
await writeJson(input.stagedDir, `tables/${acc.table}.json`, toStagedTable(acc, now));
|
||||
for (const [table, acc] of [...byTable.entries()].sort(([left], [right]) => left.localeCompare(right))) {
|
||||
await writeJson(input.stagedDir, `tables/${table}.json`, toStagedTable(acc, now));
|
||||
}
|
||||
const patternsInput = toPatternsInput(parsedTemplates);
|
||||
const patternInputSplit = splitHistoricSqlPatternInputs(patternsInput);
|
||||
const allWarnings = [...new Set([...warnings, ...patternInputSplit.warnings])];
|
||||
const allWarnings = [...warnings, ...patternInputSplit.warnings];
|
||||
await writeJson(input.stagedDir, 'patterns-input.json', patternInputSplit.auditInput);
|
||||
for (const shard of patternInputSplit.shards) {
|
||||
await writeJson(input.stagedDir, shard.path, shard.input);
|
||||
|
|
|
|||
|
|
@ -8,22 +8,9 @@ export type HistoricSqlDialect = z.infer<typeof historicSqlDialectSchema>;
|
|||
|
||||
const filterModeSchema = z.enum(['exclude', 'include', 'mark-only']);
|
||||
|
||||
const ktxTableRefSchema = z.object({
|
||||
catalog: z.string().nullable(),
|
||||
db: z.string().nullable(),
|
||||
name: z.string().min(1),
|
||||
}).strict();
|
||||
|
||||
const ktxTableRefWithColumnsSchema = ktxTableRefSchema.extend({
|
||||
columns: z.array(z.string().min(1)).optional(),
|
||||
}).strict();
|
||||
|
||||
const historicSqlCommonPullConfigSchema = z.object({
|
||||
minExecutions: z.number().int().nonnegative().default(5),
|
||||
enabledTables: z.array(ktxTableRefSchema).default([]),
|
||||
enabledSchemas: z.array(z.string().min(1)).default([]),
|
||||
modeledTableCatalog: z.array(ktxTableRefWithColumnsSchema).default([]),
|
||||
scopeFloorWarnings: z.array(z.string()).default([]),
|
||||
enabledTables: z.array(z.string().min(1)).default([]),
|
||||
filters: z.object({
|
||||
serviceAccounts: z.object({
|
||||
patterns: z.array(z.string()).default([]),
|
||||
|
|
@ -81,7 +68,6 @@ export type AggregatedTemplate = z.infer<typeof aggregatedTemplateSchema>;
|
|||
|
||||
export const stagedTableInputSchema = z.object({
|
||||
table: z.string().min(1),
|
||||
tableRef: ktxTableRefSchema,
|
||||
stats: z.object({
|
||||
executionsBucket: z.string(),
|
||||
distinctUsersBucket: z.string(),
|
||||
|
|
@ -107,7 +93,7 @@ export const stagedPatternsInputSchema = z.object({
|
|||
templates: z.array(z.object({
|
||||
id: z.string(),
|
||||
canonicalSql: z.string(),
|
||||
tablesTouched: z.array(ktxTableRefSchema),
|
||||
tablesTouched: z.array(z.string()),
|
||||
executionsBucket: z.string(),
|
||||
distinctUsersBucket: z.string(),
|
||||
dialect: historicSqlDialectSchema,
|
||||
|
|
|
|||
|
|
@ -40,7 +40,6 @@ export interface CuratorPaginationInput {
|
|||
buildToolSet: (passNumber: number) => KtxRuntimeToolSet;
|
||||
getReconciliationActions: () => MemoryAction[];
|
||||
onStepFinish?: (info: { passNumber: number; stepIndex: number; stepBudget: number }) => void;
|
||||
abortSignal?: AbortSignal;
|
||||
}
|
||||
|
||||
interface CuratorPaginationResult extends ReconciliationOutcome {
|
||||
|
|
@ -244,7 +243,6 @@ export class CuratorPaginationService implements CuratorPaginationPort {
|
|||
sourceKey: params.input.sourceKey,
|
||||
jobId: params.input.jobId,
|
||||
forceRun: params.forceRun,
|
||||
abortSignal: params.input.abortSignal,
|
||||
onStepFinish: params.input.onStepFinish
|
||||
? ({ stepIndex, stepBudget }) =>
|
||||
params.input.onStepFinish?.({ passNumber: params.passNumber, stepIndex, stepBudget })
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@ export interface RepairFinalGateFailureInput {
|
|||
repairKind: FinalGateRepairKind;
|
||||
maxAttempts?: number;
|
||||
stepBudget?: number;
|
||||
abortSignal?: AbortSignal;
|
||||
}
|
||||
|
||||
const readRepairFileSchema = z.object({
|
||||
|
|
@ -201,7 +200,6 @@ export async function repairFinalGateFailure(
|
|||
jobId: input.trace.context.jobId,
|
||||
repairKind: input.repairKind,
|
||||
},
|
||||
abortSignal: input.abortSignal,
|
||||
}),
|
||||
);
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ import { dirname, join } from 'node:path';
|
|||
import pLimit from 'p-limit';
|
||||
import { z } from 'zod';
|
||||
import { type KtxLogger, noopLogger } from '../../context/core/config.js';
|
||||
import type { RateLimitWaitState } from '../../context/llm/rate-limit-governor.js';
|
||||
import { createRuntimeToolDescriptorFromAiTool } from '../../context/llm/runtime-tools.js';
|
||||
import type { KtxRuntimeToolSet } from '../../context/llm/runtime-port.js';
|
||||
import type { CaptureSession, MemoryAction } from '../../context/memory/types.js';
|
||||
|
|
@ -26,7 +25,6 @@ import {
|
|||
deriveFinalizationWikiPageKeys,
|
||||
} from './finalization-scope.js';
|
||||
import { FileIngestTraceWriter, ingestTracePathForJob, type IngestTraceWriter, traceTimed } from './ingest-trace.js';
|
||||
import { formatIngestProfile, formatIngestProfileJson, readIngestProfile, resolveIngestProfileMode } from './ingest-profile.js';
|
||||
import { integrateWorkUnitPatch } from './isolated-diff/patch-integrator.js';
|
||||
import { resolveTextualConflict } from './isolated-diff/textual-conflict-resolver.js';
|
||||
import { runIsolatedWorkUnit } from './isolated-diff/work-unit-executor.js';
|
||||
|
|
@ -71,7 +69,7 @@ import { createEvictionListTool } from './tools/eviction-list.tool.js';
|
|||
import { createReadRawSpanTool } from './tools/read-raw-span.tool.js';
|
||||
import { createStageDiffTool } from './tools/stage-diff.tool.js';
|
||||
import { createStageListTool } from './tools/stage-list.tool.js';
|
||||
import { flushToolCallLogs, type ToolCallLogEntry, wrapToolsWithLogger } from './tools/tool-call-logger.js';
|
||||
import { type ToolCallLogEntry, wrapToolsWithLogger } from './tools/tool-call-logger.js';
|
||||
import {
|
||||
createMutableToolTranscriptSummary,
|
||||
recordToolTranscriptEntry,
|
||||
|
|
@ -220,10 +218,6 @@ export class IngestBundleRunner {
|
|||
}
|
||||
|
||||
async run(job: IngestBundleJob, ctx?: IngestJobContext): Promise<IngestBundleResult> {
|
||||
const unsubscribeRateLimitGovernor = this.subscribeRateLimitGovernor({
|
||||
trace: this.createTrace(job),
|
||||
memoryFlow: ctx?.memoryFlow,
|
||||
});
|
||||
const key = job.connectionId;
|
||||
const previous = this.chainByConnection.get(key);
|
||||
if (previous) {
|
||||
|
|
@ -245,103 +239,6 @@ export class IngestBundleRunner {
|
|||
} catch (error) {
|
||||
ctx?.memoryFlow?.finish('error', [sanitizeMemoryFlowError(error)]);
|
||||
throw error;
|
||||
} finally {
|
||||
unsubscribeRateLimitGovernor();
|
||||
await this.maybeEmitIngestProfile(job.jobId);
|
||||
}
|
||||
}
|
||||
|
||||
private formatRateLimitWait(
|
||||
state: Extract<RateLimitWaitState, { kind: 'wait_tick' | 'wait_started' | 'wait_finished' }>,
|
||||
): string {
|
||||
const seconds = Math.ceil(state.remainingMs / 1_000);
|
||||
const minutes = Math.floor(seconds / 60);
|
||||
const remainder = seconds % 60;
|
||||
const duration = minutes > 0 ? `${minutes}m${String(remainder).padStart(2, '0')}s` : `${seconds}s`;
|
||||
const type = state.rateLimitType ? ` ${state.rateLimitType}` : '';
|
||||
return `Rate-limited (${state.provider}${type}); resuming in ${duration}; Ctrl+C to stop`;
|
||||
}
|
||||
|
||||
private subscribeRateLimitGovernor(input: {
|
||||
trace: IngestTraceWriter;
|
||||
memoryFlow?: MemoryFlowEventSink;
|
||||
}): () => void {
|
||||
const governor = this.deps.settings.rateLimitGovernor;
|
||||
if (!governor) {
|
||||
return () => undefined;
|
||||
}
|
||||
return governor.subscribe((state: RateLimitWaitState) => {
|
||||
if (state.kind === 'rate_limit_observed') {
|
||||
void input.trace.event('info', 'rate_limit', 'rate_limit_observed', { ...state });
|
||||
return;
|
||||
}
|
||||
if (state.kind === 'concurrency_adjusted') {
|
||||
void input.trace.event('info', 'rate_limit', 'concurrency_adjusted', { ...state });
|
||||
return;
|
||||
}
|
||||
void input.trace.event('info', 'rate_limit', state.kind, { ...state });
|
||||
if (state.kind === 'wait_tick' || state.kind === 'wait_started') {
|
||||
input.memoryFlow?.emit({
|
||||
type: 'rate_limit_wait',
|
||||
provider: state.provider,
|
||||
...(state.rateLimitType ? { rateLimitType: state.rateLimitType } : {}),
|
||||
resumeAtMs: state.resumeAtMs,
|
||||
remainingMs: state.remainingMs,
|
||||
});
|
||||
input.memoryFlow?.emit({
|
||||
type: 'stage_progress',
|
||||
stage: 'integration',
|
||||
percent: 50,
|
||||
message: this.formatRateLimitWait(state),
|
||||
transient: true,
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private async withRateLimitWorkSlot<T>(abortSignal: AbortSignal | undefined, fn: () => Promise<T>): Promise<T> {
|
||||
const governor = this.deps.settings.rateLimitGovernor;
|
||||
if (!governor) {
|
||||
return fn();
|
||||
}
|
||||
const release = await governor.acquireWorkSlot(abortSignal);
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
release();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* When profiling is enabled — via the `KTX_PROFILE_INGEST` env var or the
|
||||
* `ingest.profile` config setting — read the job's trace + tool transcripts
|
||||
* and print a rolled-up timing breakdown to stderr. `json` emits the raw
|
||||
* structured profile for coding agents; `table` emits a human summary.
|
||||
* Best-effort: profiling never affects the run outcome.
|
||||
*/
|
||||
private async maybeEmitIngestProfile(jobId: string): Promise<void> {
|
||||
const mode = resolveIngestProfileMode(this.deps.settings.profileIngest);
|
||||
if (mode === 'off') {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
// Tool transcripts are appended fire-and-forget; flush them so per-work-unit
|
||||
// toolMs (and the derived model-vs-tool split) is complete before we read.
|
||||
await flushToolCallLogs();
|
||||
const storage = this.deps.storage as typeof this.deps.storage & {
|
||||
resolveTracePath?: (jobId: string) => string;
|
||||
};
|
||||
const profile = await readIngestProfile(jobId, {
|
||||
tracePath: storage.resolveTracePath?.(jobId) ?? ingestTracePathForJob(this.deps.storage.homeDir, jobId),
|
||||
transcriptDir: this.deps.storage.resolveTranscriptDir(jobId),
|
||||
});
|
||||
process.stderr.write(`\n${mode === 'json' ? formatIngestProfileJson(profile) : formatIngestProfile(profile)}`);
|
||||
} catch (error) {
|
||||
this.logger.warn(
|
||||
`[ingest-bundle] ingest profile unavailable for job=${jobId}: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -944,7 +841,6 @@ export class IngestBundleRunner {
|
|||
includeContextEvidenceTools: boolean;
|
||||
currentTableExists(tableRef: string): Promise<boolean>;
|
||||
memoryFlow?: MemoryFlowEventSink;
|
||||
abortSignal?: AbortSignal;
|
||||
wuSkillNames: string[];
|
||||
onStepFinish?: (info: { stepIndex: number; stepBudget: number }) => void;
|
||||
}): Promise<WorkUnitOutcome> {
|
||||
|
|
@ -1097,7 +993,6 @@ export class IngestBundleRunner {
|
|||
jobId: input.job.jobId,
|
||||
toolFailureCount: (unitKey) => input.transcriptSummaries.get(unitKey)?.fatalErrorCount ?? 0,
|
||||
onStepFinish: input.onStepFinish,
|
||||
abortSignal: input.abortSignal,
|
||||
},
|
||||
input.wu,
|
||||
);
|
||||
|
|
@ -1205,15 +1100,8 @@ export class IngestBundleRunner {
|
|||
|
||||
const scopeDescriptor = adapter.describeScope ? await adapter.describeScope(stagedDir) : null;
|
||||
|
||||
const sessionWorktree = await traceTimed(
|
||||
trace,
|
||||
'worktree',
|
||||
'session_worktree_created',
|
||||
{ jobId: job.jobId },
|
||||
() =>
|
||||
this.deps.lockingService.withLock('config:repo', () =>
|
||||
this.deps.sessionWorktreeService.create(job.jobId, baseSha),
|
||||
),
|
||||
const sessionWorktree = await this.deps.lockingService.withLock('config:repo', () =>
|
||||
this.deps.sessionWorktreeService.create(job.jobId, baseSha),
|
||||
);
|
||||
let cleanupOutcome: 'success' | 'crash' | 'conflict' = 'crash';
|
||||
|
||||
|
|
@ -1384,34 +1272,26 @@ export class IngestBundleRunner {
|
|||
sourceContextReport = chunk.contextReport;
|
||||
parseArtifacts = chunk.parseArtifacts;
|
||||
reconcileNotes = chunk.reconcileNotes ?? [];
|
||||
const pageTriage = this.deps.pageTriage;
|
||||
const triageRunId = runRow.id;
|
||||
triageResult =
|
||||
contextReport && adapter.triageSupported && pageTriage
|
||||
? await traceTimed(runTrace, 'triage', 'page_triage', { sourceKey: job.sourceKey }, () =>
|
||||
pageTriage.triageRun({
|
||||
stagedDir,
|
||||
runId: triageRunId,
|
||||
connectionId: job.connectionId,
|
||||
sourceKey: job.sourceKey,
|
||||
syncId,
|
||||
jobId: job.jobId,
|
||||
diffSet,
|
||||
adapter,
|
||||
}),
|
||||
)
|
||||
contextReport && adapter.triageSupported && this.deps.pageTriage
|
||||
? await this.deps.pageTriage.triageRun({
|
||||
stagedDir,
|
||||
runId: runRow.id,
|
||||
connectionId: job.connectionId,
|
||||
sourceKey: job.sourceKey,
|
||||
syncId,
|
||||
jobId: job.jobId,
|
||||
diffSet,
|
||||
adapter,
|
||||
})
|
||||
: null;
|
||||
workUnits = this.filterWorkUnitsForTriage(workUnits, triageResult);
|
||||
const clusterWorkUnits = adapter.clusterWorkUnits;
|
||||
if (clusterWorkUnits && workUnits.length > 0) {
|
||||
const preClusterCount = workUnits.length;
|
||||
workUnits = await traceTimed(
|
||||
runTrace,
|
||||
'clustering',
|
||||
'cluster_work_units',
|
||||
{ workUnitCount: preClusterCount },
|
||||
() => clusterWorkUnits({ workUnits, stagedDir, embedding: this.deps.embedding }),
|
||||
);
|
||||
if (adapter.clusterWorkUnits && workUnits.length > 0) {
|
||||
workUnits = await adapter.clusterWorkUnits({
|
||||
workUnits,
|
||||
stagedDir,
|
||||
embedding: this.deps.embedding,
|
||||
});
|
||||
}
|
||||
await stage2?.updateProgress(1.0, `Planned ${workUnits.length} update${workUnits.length === 1 ? '' : 's'}`);
|
||||
}
|
||||
|
|
@ -1446,13 +1326,7 @@ export class IngestBundleRunner {
|
|||
});
|
||||
|
||||
// Build shared per-job context.
|
||||
const [wikiIndex, slIndex] = await traceTimed(
|
||||
runTrace,
|
||||
'index_build',
|
||||
'build_indexes',
|
||||
{ connectionCount: slConnectionIds.length },
|
||||
() => Promise.all([this.buildWikiIndex(), this.buildSlIndex(slConnectionIds)]),
|
||||
);
|
||||
const [wikiIndex, slIndex] = await Promise.all([this.buildWikiIndex(), this.buildSlIndex(slConnectionIds)]);
|
||||
|
||||
const baseFraming = await this.deps.promptService.loadPrompt('memory_agent_bundle_ingest_work_unit');
|
||||
const wuSkillNames = Array.from(
|
||||
|
|
@ -1593,8 +1467,7 @@ export class IngestBundleRunner {
|
|||
try {
|
||||
await Promise.all(
|
||||
workUnits.map((wu, index) =>
|
||||
limitWorkUnit(() =>
|
||||
this.withRateLimitWorkSlot(ctx?.abortSignal, async () => {
|
||||
limitWorkUnit(async () => {
|
||||
const outcome = await runIsolatedWorkUnit({
|
||||
unitIndex: index,
|
||||
ingestionBaseSha,
|
||||
|
|
@ -1602,7 +1475,6 @@ export class IngestBundleRunner {
|
|||
patchDir,
|
||||
trace: runTrace,
|
||||
workUnit: wu,
|
||||
abortSignal: ctx?.abortSignal,
|
||||
afterSuccess: (child) => copyTransientIngestEvidence(child.workdir, sessionWorktree.workdir),
|
||||
run: async (child) => {
|
||||
const scopedWikiService = this.deps.wikiService.forWorktree(child.workdir);
|
||||
|
|
@ -1636,7 +1508,6 @@ export class IngestBundleRunner {
|
|||
includeContextEvidenceTools: adapter.evidenceIndexing === 'documents' && !!contextReport,
|
||||
currentTableExists: (tableRef) =>
|
||||
this.tableRefExistsInSemanticLayer(scopedSemanticLayerService, slConnectionIds, tableRef),
|
||||
abortSignal: ctx?.abortSignal,
|
||||
memoryFlow,
|
||||
wuSkillNames,
|
||||
onStepFinish: ({ stepIndex, stepBudget }) => {
|
||||
|
|
@ -1666,8 +1537,7 @@ export class IngestBundleRunner {
|
|||
completedWorkUnits / workUnits.length,
|
||||
`${completedWorkUnits} of ${workUnits.length} work units complete`,
|
||||
);
|
||||
}),
|
||||
),
|
||||
}),
|
||||
),
|
||||
);
|
||||
} catch (error) {
|
||||
|
|
@ -1766,7 +1636,6 @@ export class IngestBundleRunner {
|
|||
reason: context.reason,
|
||||
maxAttempts: 1,
|
||||
stepBudget: 12,
|
||||
abortSignal: ctx?.abortSignal,
|
||||
});
|
||||
emitStageProgress(
|
||||
'integration',
|
||||
|
|
@ -1788,7 +1657,6 @@ export class IngestBundleRunner {
|
|||
repairKind: 'patch_semantic_gate',
|
||||
maxAttempts: 1,
|
||||
stepBudget: 16,
|
||||
abortSignal: ctx?.abortSignal,
|
||||
});
|
||||
emitStageProgress(
|
||||
'integration',
|
||||
|
|
@ -2013,8 +1881,6 @@ export class IngestBundleRunner {
|
|||
let curatorWarnings: string[] = [];
|
||||
let reconcileOutcome: Awaited<ReturnType<typeof runReconciliationStage4>>;
|
||||
|
||||
const reconcileStartedAt = Date.now();
|
||||
const reconcileMode = contextReport && this.deps.curatorPagination ? 'curator' : 'single';
|
||||
if (contextReport && this.deps.curatorPagination) {
|
||||
const curatorOutcome = await this.deps.curatorPagination.reconcile({
|
||||
runId: runRow.id,
|
||||
|
|
@ -2068,7 +1934,6 @@ export class IngestBundleRunner {
|
|||
);
|
||||
}
|
||||
: undefined,
|
||||
abortSignal: ctx?.abortSignal,
|
||||
});
|
||||
curatorReport = curatorOutcome.report;
|
||||
curatorWarnings = curatorOutcome.warnings;
|
||||
|
|
@ -2114,7 +1979,6 @@ export class IngestBundleRunner {
|
|||
sourceKey: job.sourceKey,
|
||||
jobId: job.jobId,
|
||||
force: !!overrideReport,
|
||||
abortSignal: ctx?.abortSignal,
|
||||
onStepFinish: stage4
|
||||
? ({ stepIndex, stepBudget }) => {
|
||||
emitStageProgress('reconciliation', 85, `Reconciling results: step ${stepIndex}/${stepBudget}`, {
|
||||
|
|
@ -2125,33 +1989,6 @@ export class IngestBundleRunner {
|
|||
: undefined,
|
||||
});
|
||||
}
|
||||
await runTrace.event(
|
||||
'debug',
|
||||
'reconciliation',
|
||||
'reconciliation_executed',
|
||||
{
|
||||
mode: reconcileMode,
|
||||
skipped: reconcileOutcome.skipped,
|
||||
...(reconcileOutcome.stopReason ? { stopReason: reconcileOutcome.stopReason } : {}),
|
||||
...(reconcileOutcome.metrics
|
||||
? {
|
||||
agentLoopMs: reconcileOutcome.metrics.totalMs,
|
||||
stepCount: reconcileOutcome.metrics.stepCount,
|
||||
...(reconcileOutcome.metrics.usage.inputTokens !== undefined
|
||||
? { inputTokens: reconcileOutcome.metrics.usage.inputTokens }
|
||||
: {}),
|
||||
...(reconcileOutcome.metrics.usage.outputTokens !== undefined
|
||||
? { outputTokens: reconcileOutcome.metrics.usage.outputTokens }
|
||||
: {}),
|
||||
...(reconcileOutcome.metrics.usage.totalTokens !== undefined
|
||||
? { totalTokens: reconcileOutcome.metrics.usage.totalTokens }
|
||||
: {}),
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
undefined,
|
||||
Date.now() - reconcileStartedAt,
|
||||
);
|
||||
latestReconciliationSkipped = reconcileOutcome.skipped;
|
||||
|
||||
const danglingReconcileWikiRefs = await findDanglingWikiRefsForActions({
|
||||
|
|
@ -2199,7 +2036,6 @@ export class IngestBundleRunner {
|
|||
activePhase = 'finalization';
|
||||
if (adapter.finalize) {
|
||||
const stageFinalization = ctx?.startPhase(0.04);
|
||||
const finalizationStartedAt = Date.now();
|
||||
emitStageProgress('finalization', 87, 'Running deterministic finalization');
|
||||
await stageFinalization?.updateProgress(0.0, 'Running deterministic finalization');
|
||||
await runTrace.event('debug', 'finalization', 'finalization_started', { sourceKey: job.sourceKey });
|
||||
|
|
@ -2379,21 +2215,14 @@ export class IngestBundleRunner {
|
|||
latestFinalizationOutcome = finalizationOutcome;
|
||||
emitStageProgress('finalization', 88, 'Deterministic finalization complete');
|
||||
await stageFinalization?.updateProgress(1.0, 'Deterministic finalization complete');
|
||||
await runTrace.event(
|
||||
'debug',
|
||||
'finalization',
|
||||
'finalization_committed',
|
||||
{
|
||||
sourceKey: job.sourceKey,
|
||||
commitSha: finalizationSha,
|
||||
touchedPaths: finalizationTouchedPaths,
|
||||
touchedSources: finalizationTouchedSources,
|
||||
changedWikiPageKeys: finalizationChangedWikiPageKeys,
|
||||
warnings: result.warnings,
|
||||
},
|
||||
undefined,
|
||||
Date.now() - finalizationStartedAt,
|
||||
);
|
||||
await runTrace.event('debug', 'finalization', 'finalization_committed', {
|
||||
sourceKey: job.sourceKey,
|
||||
commitSha: finalizationSha,
|
||||
touchedPaths: finalizationTouchedPaths,
|
||||
touchedSources: finalizationTouchedSources,
|
||||
changedWikiPageKeys: finalizationChangedWikiPageKeys,
|
||||
warnings: result.warnings,
|
||||
});
|
||||
} else {
|
||||
await runTrace.event('debug', 'finalization', 'finalization_skipped', { sourceKey: job.sourceKey });
|
||||
}
|
||||
|
|
@ -2547,7 +2376,6 @@ export class IngestBundleRunner {
|
|||
repairKind: 'final_artifact_gate',
|
||||
maxAttempts: 1,
|
||||
stepBudget: 16,
|
||||
abortSignal: ctx?.abortSignal,
|
||||
});
|
||||
|
||||
isolatedDiffSummary.gateRepairAttempts += gateRepair.attempts;
|
||||
|
|
@ -2676,7 +2504,6 @@ export class IngestBundleRunner {
|
|||
const stage6 = ctx?.startPhase(0.04);
|
||||
emitStageProgress('save', 91, 'Saving changes');
|
||||
await stage6?.updateProgress(0.0, 'Saving changes');
|
||||
const squashStartedAt = Date.now();
|
||||
try {
|
||||
await sessionWorktree.git.assertWorktreeClean();
|
||||
} catch (error) {
|
||||
|
|
@ -2700,17 +2527,10 @@ export class IngestBundleRunner {
|
|||
throw new Error(`squash merge conflict: ${mergeResult.conflictPaths.join(', ')}`);
|
||||
}
|
||||
const commitSha = mergeResult.touchedPaths.length === 0 ? null : mergeResult.squashSha;
|
||||
await runTrace.event(
|
||||
'debug',
|
||||
'squash',
|
||||
'squash_finished',
|
||||
{
|
||||
commitSha,
|
||||
touchedPaths: mergeResult.touchedPaths,
|
||||
},
|
||||
undefined,
|
||||
Date.now() - squashStartedAt,
|
||||
);
|
||||
await runTrace.event('debug', 'squash', 'squash_finished', {
|
||||
commitSha,
|
||||
touchedPaths: mergeResult.touchedPaths,
|
||||
});
|
||||
const memoryFlowSavedActions = stageIndex.workUnits
|
||||
.flatMap((wu) => wu.actions)
|
||||
.concat(reconcileActions)
|
||||
|
|
@ -2727,7 +2547,6 @@ export class IngestBundleRunner {
|
|||
// transaction. If this throws, the run fails and no partial index state
|
||||
// survives (thanks to the transactional upsert in applyDiffTransactional).
|
||||
if (commitSha) {
|
||||
const indexSyncStartedAt = Date.now();
|
||||
// Multi-file squash → omit path so the handler diffs the whole commit
|
||||
// (a comma-joined pathspec would match nothing and the job would no-op).
|
||||
const pathFilter = mergeResult.touchedPaths.length === 1 ? mergeResult.touchedPaths[0] : '';
|
||||
|
|
@ -2752,14 +2571,6 @@ export class IngestBundleRunner {
|
|||
);
|
||||
}
|
||||
}
|
||||
await runTrace.event(
|
||||
'debug',
|
||||
'index_sync',
|
||||
'post_squash_index_sync_finished',
|
||||
{ connectionCount: touchedConnections.length },
|
||||
undefined,
|
||||
Date.now() - indexSyncStartedAt,
|
||||
);
|
||||
}
|
||||
|
||||
const stage5 = ctx?.startPhase(0.04);
|
||||
|
|
|
|||
|
|
@ -1,437 +0,0 @@
|
|||
import { readdir, readFile } from 'node:fs/promises';
|
||||
import { join } from 'node:path';
|
||||
import { z } from 'zod';
|
||||
|
||||
export interface IngestProfilePaths {
|
||||
tracePath: string;
|
||||
transcriptDir: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Post-processor over the ingest trace (`<home>/ingest-traces/<jobId>/trace.jsonl`)
|
||||
* and per-work-unit tool transcripts. Turns the durations recorded during a run
|
||||
* into a rolled-up "where did the time go" view. Gated for display by
|
||||
* `KTX_PROFILE_INGEST`; the durations themselves are always written to the trace.
|
||||
*/
|
||||
|
||||
const traceEventSchema = z
|
||||
.object({
|
||||
at: z.string().optional(),
|
||||
phase: z.string(),
|
||||
event: z.string(),
|
||||
durationMs: z.number().optional(),
|
||||
data: z.record(z.string(), z.unknown()).optional(),
|
||||
})
|
||||
.loose();
|
||||
|
||||
/** @internal */
|
||||
export type ProfiledTraceEvent = z.infer<typeof traceEventSchema>;
|
||||
|
||||
export interface IngestProfile {
|
||||
jobId: string;
|
||||
totalWallMs?: number;
|
||||
phases: Array<{
|
||||
phase: string;
|
||||
totalMs: number;
|
||||
/** Number of timed (durationMs-bearing) events that contributed to this phase. */
|
||||
count: number;
|
||||
}>;
|
||||
workUnits: Array<{
|
||||
unitKey: string;
|
||||
status?: string;
|
||||
/** Wall-clock for the whole work-unit run (agent loop + validation + git). */
|
||||
totalMs?: number;
|
||||
/** Pure `generateText` agent-loop time reported by the runtime. */
|
||||
agentLoopMs?: number;
|
||||
/** Summed tool-execution time from the work-unit transcript. */
|
||||
toolMs?: number;
|
||||
/** Derived model "thinking" time = agentLoopMs - toolMs (clamped at 0). */
|
||||
modelMs?: number;
|
||||
/** Worktree create time. */
|
||||
createMs?: number;
|
||||
/** Worktree teardown time. */
|
||||
cleanupMs?: number;
|
||||
stepCount?: number;
|
||||
totalTokens?: number;
|
||||
}>;
|
||||
workUnitCount: number;
|
||||
failedWorkUnitCount: number;
|
||||
/**
|
||||
* Plain-language diagnosis plus the raw numbers behind it, so a reader (human
|
||||
* or coding agent) gets the conclusion without re-deriving it from the tables.
|
||||
*/
|
||||
summary: {
|
||||
/** One-sentence conclusion, e.g. which phase dominated and whether work was model- or tool-bound. */
|
||||
headline: string;
|
||||
dominantPhase?: { phase: string; totalMs: number; pctOfWall?: number };
|
||||
/** Aggregate across all work units, in milliseconds. */
|
||||
workUnits?: {
|
||||
count: number;
|
||||
failed: number;
|
||||
agentLoopMs: number;
|
||||
modelMs: number;
|
||||
toolMs: number;
|
||||
/** Percent of agent-loop time spent in model generation vs tool execution. */
|
||||
modelPct?: number;
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
type IngestWorkUnitTiming = IngestProfile['workUnits'][number];
|
||||
|
||||
function asNumber(value: unknown): number | undefined {
|
||||
return typeof value === 'number' && Number.isFinite(value) ? value : undefined;
|
||||
}
|
||||
|
||||
function asString(value: unknown): string | undefined {
|
||||
return typeof value === 'string' ? value : undefined;
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function parseTraceEvents(traceText: string): ProfiledTraceEvent[] {
|
||||
const events: ProfiledTraceEvent[] = [];
|
||||
for (const line of traceText.split('\n')) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed) {
|
||||
continue;
|
||||
}
|
||||
let json: unknown;
|
||||
try {
|
||||
json = JSON.parse(trimmed);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
const parsed = traceEventSchema.safeParse(json);
|
||||
if (parsed.success) {
|
||||
events.push(parsed.data);
|
||||
}
|
||||
}
|
||||
return events;
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export function aggregateIngestProfile(input: {
|
||||
jobId: string;
|
||||
events: ProfiledTraceEvent[];
|
||||
toolMsByUnit: Record<string, number>;
|
||||
}): IngestProfile {
|
||||
const { jobId, events, toolMsByUnit } = input;
|
||||
|
||||
const phaseTotals = new Map<string, { totalMs: number; count: number }>();
|
||||
const workUnits = new Map<string, IngestWorkUnitTiming>();
|
||||
|
||||
const wu = (unitKey: string): IngestWorkUnitTiming => {
|
||||
let existing = workUnits.get(unitKey);
|
||||
if (!existing) {
|
||||
existing = { unitKey };
|
||||
workUnits.set(unitKey, existing);
|
||||
}
|
||||
return existing;
|
||||
};
|
||||
|
||||
let minAt = Number.POSITIVE_INFINITY;
|
||||
let maxAt = Number.NEGATIVE_INFINITY;
|
||||
|
||||
for (const event of events) {
|
||||
const at = event.at ? Date.parse(event.at) : Number.NaN;
|
||||
if (!Number.isNaN(at)) {
|
||||
minAt = Math.min(minAt, at);
|
||||
maxAt = Math.max(maxAt, at);
|
||||
}
|
||||
|
||||
if (event.durationMs !== undefined) {
|
||||
const bucket = phaseTotals.get(event.phase) ?? { totalMs: 0, count: 0 };
|
||||
bucket.totalMs += event.durationMs;
|
||||
bucket.count += 1;
|
||||
phaseTotals.set(event.phase, bucket);
|
||||
}
|
||||
|
||||
const data = event.data ?? {};
|
||||
const unitKey = asString(data.unitKey);
|
||||
if (unitKey) {
|
||||
const entry = wu(unitKey);
|
||||
if (event.event === 'work_unit_executed') {
|
||||
entry.totalMs = event.durationMs;
|
||||
entry.agentLoopMs = asNumber(data.agentLoopMs);
|
||||
entry.stepCount = asNumber(data.stepCount);
|
||||
entry.totalTokens = asNumber(data.totalTokens);
|
||||
entry.status = asString(data.status) ?? entry.status;
|
||||
} else if (event.event === 'work_unit_child_created') {
|
||||
entry.createMs = event.durationMs;
|
||||
} else if (event.event === 'work_unit_child_cleanup') {
|
||||
entry.cleanupMs = event.durationMs;
|
||||
} else if (event.event === 'work_unit_failed_before_patch') {
|
||||
entry.status = entry.status ?? 'failed';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const [unitKey, entry] of workUnits) {
|
||||
const toolMs = toolMsByUnit[unitKey];
|
||||
if (toolMs !== undefined) {
|
||||
entry.toolMs = toolMs;
|
||||
if (entry.agentLoopMs !== undefined) {
|
||||
entry.modelMs = Math.max(0, entry.agentLoopMs - toolMs);
|
||||
}
|
||||
} else if (entry.agentLoopMs !== undefined) {
|
||||
entry.modelMs = entry.agentLoopMs;
|
||||
}
|
||||
}
|
||||
|
||||
const phases = [...phaseTotals.entries()]
|
||||
.map(([phase, { totalMs, count }]) => ({ phase, totalMs, count }))
|
||||
.sort((a, b) => b.totalMs - a.totalMs);
|
||||
|
||||
const workUnitList = [...workUnits.values()].sort((a, b) => (b.totalMs ?? 0) - (a.totalMs ?? 0));
|
||||
const totalWallMs = Number.isFinite(minAt) && Number.isFinite(maxAt) && maxAt >= minAt ? maxAt - minAt : undefined;
|
||||
const failedWorkUnitCount = workUnitList.filter((entry) => entry.status === 'failed').length;
|
||||
|
||||
return {
|
||||
jobId,
|
||||
...(totalWallMs !== undefined ? { totalWallMs } : {}),
|
||||
phases,
|
||||
workUnits: workUnitList,
|
||||
workUnitCount: workUnitList.length,
|
||||
failedWorkUnitCount,
|
||||
summary: buildSummary(phases, workUnitList, failedWorkUnitCount, totalWallMs),
|
||||
};
|
||||
}
|
||||
|
||||
function buildSummary(
|
||||
phases: IngestProfile['phases'],
|
||||
workUnits: IngestWorkUnitTiming[],
|
||||
failed: number,
|
||||
totalWallMs: number | undefined,
|
||||
): IngestProfile['summary'] {
|
||||
const dominant = phases[0];
|
||||
const dominantPhase = dominant
|
||||
? {
|
||||
phase: dominant.phase,
|
||||
totalMs: dominant.totalMs,
|
||||
...(totalWallMs && totalWallMs > 0
|
||||
? { pctOfWall: Math.round((dominant.totalMs / totalWallMs) * 100) }
|
||||
: {}),
|
||||
}
|
||||
: undefined;
|
||||
|
||||
const agentLoopMs = workUnits.reduce((sum, wu) => sum + (wu.agentLoopMs ?? 0), 0);
|
||||
const toolMs = workUnits.reduce((sum, wu) => sum + (wu.toolMs ?? 0), 0);
|
||||
const modelMs = workUnits.reduce((sum, wu) => sum + (wu.modelMs ?? 0), 0);
|
||||
const workUnitAggregate =
|
||||
workUnits.length > 0
|
||||
? {
|
||||
count: workUnits.length,
|
||||
failed,
|
||||
agentLoopMs,
|
||||
modelMs,
|
||||
toolMs,
|
||||
...(agentLoopMs > 0 ? { modelPct: Math.round((modelMs / agentLoopMs) * 100) } : {}),
|
||||
}
|
||||
: undefined;
|
||||
|
||||
const parts: string[] = [];
|
||||
if (dominantPhase) {
|
||||
const pct = dominantPhase.pctOfWall !== undefined ? `, ${dominantPhase.pctOfWall}% of wall time` : '';
|
||||
parts.push(`Slowest phase: ${dominantPhase.phase} (${formatMs(dominantPhase.totalMs)}${pct})`);
|
||||
}
|
||||
if (workUnitAggregate) {
|
||||
const split =
|
||||
workUnitAggregate.modelPct !== undefined
|
||||
? `, ~${workUnitAggregate.modelPct}% model generation vs ~${100 - workUnitAggregate.modelPct}% tools`
|
||||
: '';
|
||||
parts.push(
|
||||
`${workUnitAggregate.count} work unit${workUnitAggregate.count === 1 ? '' : 's'}${
|
||||
failed > 0 ? ` (${failed} failed)` : ''
|
||||
}${split}`,
|
||||
);
|
||||
}
|
||||
const headline = parts.length > 0 ? parts.join('. ') + '.' : 'No timed phases recorded.';
|
||||
|
||||
return {
|
||||
headline,
|
||||
...(dominantPhase ? { dominantPhase } : {}),
|
||||
...(workUnitAggregate ? { workUnits: workUnitAggregate } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
/** Read the trace and tool transcripts for a job and aggregate them into a profile. */
|
||||
export async function readIngestProfile(
|
||||
jobId: string,
|
||||
paths: IngestProfilePaths,
|
||||
): Promise<IngestProfile> {
|
||||
const traceText = await readFile(paths.tracePath, 'utf-8');
|
||||
const events = parseTraceEvents(traceText);
|
||||
const toolMsByUnit = await readToolMsByUnit(paths.transcriptDir);
|
||||
return aggregateIngestProfile({ jobId, events, toolMsByUnit });
|
||||
}
|
||||
|
||||
async function listTranscriptFiles(dir: string): Promise<string[]> {
|
||||
// Work-unit keys can contain slashes (e.g. "cards/marketing"), so the runner
|
||||
// writes nested transcript files (".../cards/marketing.jsonl"). Walk
|
||||
// recursively and bucket by the `wuKey` field inside each entry rather than
|
||||
// by file name.
|
||||
const entries = await readdir(dir, { withFileTypes: true }).catch(() => null);
|
||||
if (!entries) {
|
||||
return [];
|
||||
}
|
||||
const files: string[] = [];
|
||||
for (const entry of entries) {
|
||||
const full = join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
files.push(...(await listTranscriptFiles(full)));
|
||||
} else if (entry.isFile() && entry.name.endsWith('.jsonl')) {
|
||||
files.push(full);
|
||||
}
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
async function readToolMsByUnit(transcriptDir: string): Promise<Record<string, number>> {
|
||||
const toolMs: Record<string, number> = {};
|
||||
for (const file of await listTranscriptFiles(transcriptDir)) {
|
||||
let text: string;
|
||||
try {
|
||||
text = await readFile(file, 'utf-8');
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
for (const line of text.split('\n')) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const entry = JSON.parse(trimmed) as { wuKey?: unknown; durationMs?: unknown };
|
||||
const wuKey = asString(entry.wuKey);
|
||||
const ms = asNumber(entry.durationMs);
|
||||
if (wuKey && ms !== undefined) {
|
||||
toolMs[wuKey] = (toolMs[wuKey] ?? 0) + ms;
|
||||
}
|
||||
} catch {
|
||||
// skip malformed line
|
||||
}
|
||||
}
|
||||
}
|
||||
return toolMs;
|
||||
}
|
||||
|
||||
function formatMs(ms: number | undefined): string {
|
||||
if (ms === undefined) {
|
||||
return '—';
|
||||
}
|
||||
if (ms < 1000) {
|
||||
return `${Math.round(ms)}ms`;
|
||||
}
|
||||
const seconds = ms / 1000;
|
||||
if (seconds < 60) {
|
||||
return `${seconds.toFixed(1)}s`;
|
||||
}
|
||||
const minutes = Math.floor(seconds / 60);
|
||||
const rem = Math.round(seconds - minutes * 60);
|
||||
return `${minutes}m ${String(rem).padStart(2, '0')}s`;
|
||||
}
|
||||
|
||||
function formatTokens(tokens: number | undefined): string {
|
||||
if (tokens === undefined) {
|
||||
return '—';
|
||||
}
|
||||
if (tokens < 1000) {
|
||||
return String(tokens);
|
||||
}
|
||||
return `${(tokens / 1000).toFixed(1)}k`;
|
||||
}
|
||||
|
||||
function pad(value: string, width: number): string {
|
||||
return value.length >= width ? value : value + ' '.repeat(width - value.length);
|
||||
}
|
||||
|
||||
function padStart(value: string, width: number): string {
|
||||
return value.length >= width ? value : ' '.repeat(width - value.length) + value;
|
||||
}
|
||||
|
||||
/** Render a human-readable profile table for stderr / the admin command. */
|
||||
export function formatIngestProfile(profile: IngestProfile, options: { topWorkUnits?: number } = {}): string {
|
||||
const topWorkUnits = options.topWorkUnits ?? 10;
|
||||
const lines: string[] = [];
|
||||
lines.push(`ktx ingest profile — job ${profile.jobId}`);
|
||||
if (profile.totalWallMs !== undefined) {
|
||||
lines.push(` total wall time: ${formatMs(profile.totalWallMs)}`);
|
||||
}
|
||||
lines.push(` ${profile.summary.headline}`);
|
||||
|
||||
const wall = profile.totalWallMs;
|
||||
lines.push('');
|
||||
lines.push(' Phase breakdown (by total duration):');
|
||||
if (profile.phases.length === 0) {
|
||||
lines.push(' (no timed phases recorded)');
|
||||
}
|
||||
for (const phase of profile.phases) {
|
||||
const pct = wall && wall > 0 ? `(${((phase.totalMs / wall) * 100).toFixed(1)}%)` : '';
|
||||
lines.push(
|
||||
` ${pad(phase.phase, 22)}${padStart(formatMs(phase.totalMs), 9)} ${padStart(pct, 8)} ${padStart(
|
||||
String(phase.count),
|
||||
4,
|
||||
)} event${phase.count === 1 ? '' : 's'}`,
|
||||
);
|
||||
}
|
||||
|
||||
if (profile.workUnits.length > 0) {
|
||||
lines.push('');
|
||||
lines.push(` Work units (top ${Math.min(topWorkUnits, profile.workUnits.length)} slowest):`);
|
||||
lines.push(
|
||||
` ${pad('unitKey', 30)}${padStart('total', 9)}${padStart('model', 9)}${padStart('tool', 9)}${padStart(
|
||||
'steps',
|
||||
8,
|
||||
)}${padStart('tokens', 9)} status`,
|
||||
);
|
||||
for (const entry of profile.workUnits.slice(0, topWorkUnits)) {
|
||||
const steps = entry.stepCount !== undefined ? String(entry.stepCount) : '—';
|
||||
lines.push(
|
||||
` ${pad(entry.unitKey.slice(0, 30), 30)}${padStart(formatMs(entry.totalMs), 9)}${padStart(
|
||||
formatMs(entry.modelMs),
|
||||
9,
|
||||
)}${padStart(formatMs(entry.toolMs), 9)}${padStart(steps, 8)}${padStart(
|
||||
formatTokens(entry.totalTokens),
|
||||
9,
|
||||
)} ${entry.status ?? '—'}`,
|
||||
);
|
||||
}
|
||||
lines.push(
|
||||
` (${profile.workUnitCount} work unit${profile.workUnitCount === 1 ? '' : 's'} total; ${
|
||||
profile.failedWorkUnitCount
|
||||
} failed)`,
|
||||
);
|
||||
}
|
||||
|
||||
return `${lines.join('\n')}\n`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Machine-readable rendering for coding agents: the full structured profile
|
||||
* (raw milliseconds and token counts, stable keys) as a single JSON object
|
||||
* under a stable marker line so it is easy to locate and parse in stderr.
|
||||
*/
|
||||
export function formatIngestProfileJson(profile: IngestProfile): string {
|
||||
return `ktx ingest profile (json)\n${JSON.stringify(profile, null, 2)}\n`;
|
||||
}
|
||||
|
||||
export type IngestProfileMode = 'off' | 'table' | 'json';
|
||||
|
||||
/**
|
||||
* Resolve how (and whether) to emit the ingest profile, from the
|
||||
* `ingest.profile` config value and the `KTX_PROFILE_INGEST` env var. Either
|
||||
* source may request `json` (raw, agent-friendly) or a human `table`; `json`
|
||||
* wins if either asks for it.
|
||||
*/
|
||||
export function resolveIngestProfileMode(
|
||||
configValue: boolean | 'json' | undefined,
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): IngestProfileMode {
|
||||
const envValue = env.KTX_PROFILE_INGEST;
|
||||
if (configValue === 'json' || envValue === 'json') {
|
||||
return 'json';
|
||||
}
|
||||
const wantsTable =
|
||||
configValue === true || envValue === '1' || envValue === 'true' || envValue === 'table';
|
||||
return wantsTable ? 'table' : 'off';
|
||||
}
|
||||
|
|
@ -155,103 +155,18 @@ export async function integrateWorkUnitPatch(input: IntegrateWorkUnitPatchInput)
|
|||
},
|
||||
);
|
||||
} catch (semanticError) {
|
||||
const reason = errorMessage(semanticError);
|
||||
if (preApplyHead) {
|
||||
await input.integrationGit.resetHardTo(preApplyHead);
|
||||
}
|
||||
await input.trace.event('error', 'integration', 'patch_semantic_conflict_after_textual_resolution', {
|
||||
unitKey: input.unitKey,
|
||||
patchPath: input.patchPath,
|
||||
touchedPaths: textualResolution.changedPaths,
|
||||
reason,
|
||||
reason: errorMessage(semanticError),
|
||||
});
|
||||
|
||||
// A textual conflict and a semantic-gate failure can co-occur: the resolver
|
||||
// reconciles the text but can leave wiki sl_refs pointing at measures the
|
||||
// merged source no longer defines. Recover via the same gate repair the
|
||||
// clean-apply branch uses, instead of hard-failing the whole job.
|
||||
if (input.repairGateFailure) {
|
||||
const gateRepair = await input.repairGateFailure({
|
||||
unitKey: input.unitKey,
|
||||
patchPath: input.patchPath,
|
||||
touchedPaths: textualResolution.changedPaths,
|
||||
reason,
|
||||
});
|
||||
|
||||
if (gateRepair.status !== 'failed') {
|
||||
// The resolver wrote its merge to the worktree (unstaged); the repair
|
||||
// edited a subset on top. Commit the union so neither is dropped.
|
||||
const resolvedAndRepairedPaths = [
|
||||
...new Set([...textualResolution.changedPaths, ...gateRepair.changedPaths]),
|
||||
].sort();
|
||||
try {
|
||||
await traceTimed(
|
||||
input.trace,
|
||||
'integration',
|
||||
'semantic_gate_after_gate_repair',
|
||||
{ unitKey: input.unitKey, touchedPaths: gateRepair.changedPaths },
|
||||
async () => {
|
||||
await input.validateAppliedTree(gateRepair.changedPaths);
|
||||
},
|
||||
);
|
||||
|
||||
const commit = await input.integrationGit.commitFiles(
|
||||
resolvedAndRepairedPaths,
|
||||
`ingest: resolve WorkUnit ${input.unitKey} conflict`,
|
||||
input.author.name,
|
||||
input.author.email,
|
||||
);
|
||||
if (commit.created) {
|
||||
await input.trace.event('debug', 'integration', 'patch_accepted_after_textual_resolution', {
|
||||
unitKey: input.unitKey,
|
||||
commitSha: commit.commitHash,
|
||||
touchedPaths: resolvedAndRepairedPaths,
|
||||
attempts: textualResolution.attempts,
|
||||
gateRepairAttempts: gateRepair.attempts,
|
||||
});
|
||||
return {
|
||||
status: 'accepted',
|
||||
commitSha: commit.commitHash,
|
||||
touchedPaths: resolvedAndRepairedPaths,
|
||||
textualResolution,
|
||||
gateRepair,
|
||||
};
|
||||
}
|
||||
} catch (repairValidationError) {
|
||||
if (preApplyHead) {
|
||||
await input.integrationGit.resetHardTo(preApplyHead);
|
||||
}
|
||||
await input.trace.event('error', 'integration', 'patch_semantic_conflict_after_textual_resolution', {
|
||||
unitKey: input.unitKey,
|
||||
patchPath: input.patchPath,
|
||||
touchedPaths: gateRepair.changedPaths,
|
||||
reason: errorMessage(repairValidationError),
|
||||
});
|
||||
return {
|
||||
status: 'semantic_conflict',
|
||||
reason: errorMessage(repairValidationError),
|
||||
touchedPaths: gateRepair.changedPaths,
|
||||
textualResolution,
|
||||
gateRepair,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (preApplyHead) {
|
||||
await input.integrationGit.resetHardTo(preApplyHead);
|
||||
}
|
||||
return {
|
||||
status: 'semantic_conflict',
|
||||
reason: gateRepair.status === 'failed' ? gateRepair.reason : reason,
|
||||
touchedPaths: textualResolution.changedPaths,
|
||||
textualResolution,
|
||||
gateRepair,
|
||||
};
|
||||
}
|
||||
|
||||
if (preApplyHead) {
|
||||
await input.integrationGit.resetHardTo(preApplyHead);
|
||||
}
|
||||
return {
|
||||
status: 'semantic_conflict',
|
||||
reason,
|
||||
reason: errorMessage(semanticError),
|
||||
touchedPaths: textualResolution.changedPaths,
|
||||
textualResolution,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -19,7 +19,6 @@ export interface ResolveTextualConflictInput {
|
|||
reason: string;
|
||||
maxAttempts?: number;
|
||||
stepBudget?: number;
|
||||
abortSignal?: AbortSignal;
|
||||
}
|
||||
|
||||
const readIntegrationFileSchema = z.object({
|
||||
|
|
@ -209,7 +208,6 @@ export async function resolveTextualConflict(
|
|||
jobId: input.trace.context.jobId,
|
||||
unitKey: input.unitKey,
|
||||
},
|
||||
abortSignal: input.abortSignal,
|
||||
}),
|
||||
);
|
||||
|
||||
|
|
|
|||
|
|
@ -14,7 +14,6 @@ export interface RunIsolatedWorkUnitInput {
|
|||
patchDir: string;
|
||||
trace: IngestTraceWriter;
|
||||
workUnit: WorkUnit;
|
||||
abortSignal?: AbortSignal;
|
||||
run(child: IngestSessionWorktree): Promise<WorkUnitOutcome>;
|
||||
afterSuccess?(child: IngestSessionWorktree): Promise<void>;
|
||||
}
|
||||
|
|
@ -27,52 +26,16 @@ function patchFileName(unitIndex: number, unitKey: string): string {
|
|||
export async function runIsolatedWorkUnit(input: RunIsolatedWorkUnitInput): Promise<WorkUnitOutcome> {
|
||||
const sessionKey = `${input.trace.context.jobId}-${input.workUnit.unitKey}`;
|
||||
let cleanupOutcome: SessionOutcome = 'crash';
|
||||
const createStartedAt = Date.now();
|
||||
const child = await input.sessionWorktreeService.create(sessionKey, input.ingestionBaseSha);
|
||||
await input.trace.event(
|
||||
'debug',
|
||||
'work_unit',
|
||||
'work_unit_child_created',
|
||||
{
|
||||
unitKey: input.workUnit.unitKey,
|
||||
unitIndex: input.unitIndex,
|
||||
worktreePath: child.workdir,
|
||||
baseSha: input.ingestionBaseSha,
|
||||
},
|
||||
undefined,
|
||||
Date.now() - createStartedAt,
|
||||
);
|
||||
await input.trace.event('debug', 'work_unit', 'work_unit_child_created', {
|
||||
unitKey: input.workUnit.unitKey,
|
||||
unitIndex: input.unitIndex,
|
||||
worktreePath: child.workdir,
|
||||
baseSha: input.ingestionBaseSha,
|
||||
});
|
||||
|
||||
try {
|
||||
const runStartedAt = Date.now();
|
||||
const outcome = await input.run(child);
|
||||
await input.trace.event(
|
||||
'debug',
|
||||
'work_unit',
|
||||
'work_unit_executed',
|
||||
{
|
||||
unitKey: input.workUnit.unitKey,
|
||||
unitIndex: input.unitIndex,
|
||||
status: outcome.status,
|
||||
...(outcome.metrics
|
||||
? {
|
||||
agentLoopMs: outcome.metrics.totalMs,
|
||||
stepCount: outcome.metrics.stepCount,
|
||||
...(outcome.metrics.usage.inputTokens !== undefined
|
||||
? { inputTokens: outcome.metrics.usage.inputTokens }
|
||||
: {}),
|
||||
...(outcome.metrics.usage.outputTokens !== undefined
|
||||
? { outputTokens: outcome.metrics.usage.outputTokens }
|
||||
: {}),
|
||||
...(outcome.metrics.usage.totalTokens !== undefined
|
||||
? { totalTokens: outcome.metrics.usage.totalTokens }
|
||||
: {}),
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
undefined,
|
||||
Date.now() - runStartedAt,
|
||||
);
|
||||
if (outcome.status !== 'success') {
|
||||
cleanupOutcome = 'success';
|
||||
await input.trace.event('error', 'work_unit', 'work_unit_failed_before_patch', {
|
||||
|
|
@ -112,19 +75,11 @@ export async function runIsolatedWorkUnit(input: RunIsolatedWorkUnitInput): Prom
|
|||
cleanupOutcome = 'success';
|
||||
throw error;
|
||||
} finally {
|
||||
const cleanupStartedAt = Date.now();
|
||||
await input.sessionWorktreeService.cleanup(child, cleanupOutcome);
|
||||
await input.trace.event(
|
||||
'trace',
|
||||
'work_unit',
|
||||
'work_unit_child_cleanup',
|
||||
{
|
||||
unitKey: input.workUnit.unitKey,
|
||||
outcome: cleanupOutcome,
|
||||
worktreePath: child.workdir,
|
||||
},
|
||||
undefined,
|
||||
Date.now() - cleanupStartedAt,
|
||||
);
|
||||
await input.trace.event('trace', 'work_unit', 'work_unit_child_cleanup', {
|
||||
unitKey: input.workUnit.unitKey,
|
||||
outcome: cleanupOutcome,
|
||||
worktreePath: child.workdir,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,7 +9,6 @@ import { DbtSourceAdapter } from './adapters/dbt/dbt.adapter.js';
|
|||
import { FakeSourceAdapter } from './adapters/fake/fake.adapter.js';
|
||||
import { HistoricSqlSourceAdapter } from './adapters/historic-sql/historic-sql.adapter.js';
|
||||
import { PostgresPgssReader } from './adapters/historic-sql/postgres-pgss-reader.js';
|
||||
import { resolveQueryHistoryScopeFloor } from './adapters/historic-sql/scope-floor.js';
|
||||
import {
|
||||
HISTORIC_SQL_SOURCE_KEY,
|
||||
historicSqlUnifiedPullConfigSchema,
|
||||
|
|
@ -180,39 +179,12 @@ function queryHistoryRecord(connection: unknown): Record<string, unknown> | null
|
|||
return queryHistory;
|
||||
}
|
||||
|
||||
async function queryHistoryPullConfig(
|
||||
project: KtxLocalProject,
|
||||
connectionId: string,
|
||||
connection: unknown,
|
||||
): Promise<Record<string, unknown> | null> {
|
||||
function queryHistoryPullConfig(connection: unknown): Record<string, unknown> | null {
|
||||
const queryHistory = queryHistoryRecord(connection);
|
||||
if (queryHistory?.enabled !== true || !isRecord(connection)) return null;
|
||||
const driver = String(connection.driver ?? '').toLowerCase();
|
||||
const dialect = historicSqlDialectByDriver.get(driver);
|
||||
const dialect = historicSqlDialectByDriver.get(String(connection.driver ?? '').toLowerCase());
|
||||
if (!dialect) return null;
|
||||
const scopeFloor = await resolveQueryHistoryScopeFloor({
|
||||
projectDir: project.projectDir,
|
||||
connectionId,
|
||||
driver,
|
||||
connection,
|
||||
storedQueryHistory: queryHistory,
|
||||
});
|
||||
const {
|
||||
enabled: _enabled,
|
||||
dialect: _dialect,
|
||||
enabledTables: _enabledTables,
|
||||
enabledSchemas: _enabledSchemas,
|
||||
scopeFloorWarnings: _scopeFloorWarnings,
|
||||
...stored
|
||||
} = queryHistory;
|
||||
return {
|
||||
...stored,
|
||||
dialect,
|
||||
...(scopeFloor.enabledTables.length > 0 ? { enabledTables: scopeFloor.enabledTables } : {}),
|
||||
...(scopeFloor.enabledSchemas.length > 0 ? { enabledSchemas: scopeFloor.enabledSchemas } : {}),
|
||||
...(scopeFloor.modeledTableCatalog.length > 0 ? { modeledTableCatalog: scopeFloor.modeledTableCatalog } : {}),
|
||||
...(scopeFloor.warnings.length > 0 ? { scopeFloorWarnings: scopeFloor.warnings } : {}),
|
||||
};
|
||||
return { ...queryHistory, dialect };
|
||||
}
|
||||
|
||||
function stringField(value: unknown): string | null {
|
||||
|
|
@ -273,7 +245,7 @@ export async function localPullConfigForAdapter(
|
|||
if (options.historicSqlPullConfigOverride) {
|
||||
return historicSqlUnifiedPullConfigSchema.parse(options.historicSqlPullConfigOverride);
|
||||
}
|
||||
const queryHistory = await queryHistoryPullConfig(project, connectionId, connection);
|
||||
const queryHistory = queryHistoryPullConfig(connection);
|
||||
if (!queryHistory) {
|
||||
throw new Error(`Connection "${connectionId}" does not have context.queryHistory.enabled: true`);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,7 +12,6 @@ import type { KtxSemanticLayerComputePort } from '../../context/daemon/semantic-
|
|||
import { createRuntimeToolDescriptorFromAiTool } from '../../context/llm/runtime-tools.js';
|
||||
import { createLocalKtxLlmRuntimeFromConfig } from '../../context/llm/local-config.js';
|
||||
import { KtxIngestEmbeddingPortAdapter } from '../../context/llm/embedding-port.js';
|
||||
import { createRateLimitGovernorConfig, RateLimitGovernor } from '../../context/llm/rate-limit-governor.js';
|
||||
import { RuntimeAgentRunner, type AgentRunnerPort, type KtxLlmRuntimePort, type KtxRuntimeToolSet } from '../../context/llm/runtime-port.js';
|
||||
import type { KtxEmbeddingProvider } from '../../llm/types.js';
|
||||
import type { KtxLocalProject } from '../../context/project/project.js';
|
||||
|
|
@ -612,15 +611,14 @@ function nextLocalJobId(): string {
|
|||
|
||||
function localIngestLlmProviderGuardMessage(projectDir: string): string {
|
||||
return [
|
||||
'ktx ingest requires llm.provider.backend: anthropic, vertex, gateway, claude-code, or codex, or an injected agentRunner.',
|
||||
'Configure a local Claude Code/Codex session or API-backed LLM, then rerun ingest:',
|
||||
'ktx ingest requires llm.provider.backend: anthropic, vertex, gateway, or claude-code, or an injected agentRunner.',
|
||||
'Configure a local Claude Code session or API-backed LLM, then rerun ingest:',
|
||||
` ktx setup --project-dir ${projectDir} --llm-backend claude-code --no-input`,
|
||||
` ktx setup --project-dir ${projectDir} --llm-backend codex --llm-model gpt-5.5 --no-input`,
|
||||
` ktx setup --project-dir ${projectDir} --llm-backend anthropic --anthropic-api-key-env ANTHROPIC_API_KEY --llm-model claude-sonnet-4-6 --no-input`,
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
function resolveAgentRunner(options: CreateLocalBundleIngestRuntimeOptions, rateLimitGovernor: RateLimitGovernor): {
|
||||
function resolveAgentRunner(options: CreateLocalBundleIngestRuntimeOptions): {
|
||||
agentRunner: AgentRunnerPort;
|
||||
llmRuntime?: KtxLlmRuntimePort;
|
||||
} {
|
||||
|
|
@ -629,7 +627,6 @@ function resolveAgentRunner(options: CreateLocalBundleIngestRuntimeOptions, rate
|
|||
(options.createLlmRuntime ?? createLocalKtxLlmRuntimeFromConfig)(options.project.config.llm, {
|
||||
projectDir: options.project.projectDir,
|
||||
env: process.env,
|
||||
rateLimitGovernor,
|
||||
}) ??
|
||||
undefined;
|
||||
|
||||
|
|
@ -679,13 +676,7 @@ export function createLocalBundleIngestRuntime(
|
|||
const knowledgeIndex = new LocalKnowledgeIndex(options.project, embedding);
|
||||
const knowledgeEvents = new NoopKnowledgeEventPort();
|
||||
const wikiService = new KnowledgeWikiService(rootFileStore, embedding, knowledgeIndex, options.project.git, logger);
|
||||
const rateLimitGovernor = new RateLimitGovernor(
|
||||
createRateLimitGovernorConfig({
|
||||
...options.project.config.ingest.rateLimit,
|
||||
maxConcurrency: options.project.config.ingest.workUnits.maxConcurrency,
|
||||
}),
|
||||
);
|
||||
const { agentRunner, llmRuntime } = resolveAgentRunner(options, rateLimitGovernor);
|
||||
const { agentRunner, llmRuntime } = resolveAgentRunner(options);
|
||||
const promptService = new PromptService({ promptsDir, partials: [], logger });
|
||||
const storage = new LocalIngestStorage(options.project);
|
||||
const registry = registerAdapters(options.adapters);
|
||||
|
|
@ -725,8 +716,6 @@ export function createLocalBundleIngestRuntime(
|
|||
workUnitMaxConcurrency: options.project.config.ingest.workUnits.maxConcurrency,
|
||||
workUnitStepBudget: options.project.config.ingest.workUnits.stepBudget,
|
||||
workUnitFailureMode: options.project.config.ingest.workUnits.failureMode,
|
||||
rateLimitGovernor,
|
||||
profileIngest: options.project.config.ingest.profile,
|
||||
ingestTraceLevel: ingestTraceLevelFromEnv(),
|
||||
},
|
||||
skillsRegistry: new SkillsRegistryService({ skillsDir, logger }),
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ import { cp, mkdir, rm } from 'node:fs/promises';
|
|||
import { isAbsolute, resolve } from 'node:path';
|
||||
import type { KtxSqlQueryExecutorPort } from '../../context/connections/query-executor.js';
|
||||
import type { KtxLogger } from '../../context/core/config.js';
|
||||
import { createAbortError, isAbortError } from '../../context/core/abort.js';
|
||||
import type { KtxSemanticLayerComputePort } from '../../context/daemon/semantic-layer-compute.js';
|
||||
import type { AgentRunnerPort, KtxLlmRuntimePort } from '../../context/llm/runtime-port.js';
|
||||
import type { KtxLocalProject } from '../../context/project/project.js';
|
||||
|
|
@ -14,7 +13,6 @@ import { localPullConfigForAdapter, type DefaultLocalIngestAdaptersOptions } fro
|
|||
import { createLocalBundleIngestRuntime } from './local-bundle-runtime.js';
|
||||
import type { MemoryFlowEventSink } from './memory-flow/types.js';
|
||||
import { buildSyncId } from './raw-sources-paths.js';
|
||||
import { ingestReportOutcome } from './reports.js';
|
||||
import type { IngestReportBody, IngestReportSnapshot } from './reports.js';
|
||||
import { SqliteBundleIngestStore } from './sqlite-bundle-ingest-store.js';
|
||||
import type { IngestBundleResult, IngestJobContext, IngestJobPhase, IngestTrigger, SourceAdapter } from './types.js';
|
||||
|
|
@ -37,7 +35,6 @@ export interface RunLocalIngestOptions {
|
|||
queryExecutor?: KtxSqlQueryExecutorPort;
|
||||
logger?: KtxLogger;
|
||||
embeddingProvider?: import('../../llm/types.js').KtxEmbeddingProvider | null;
|
||||
abortSignal?: AbortSignal;
|
||||
}
|
||||
|
||||
export interface LocalIngestResult {
|
||||
|
|
@ -82,7 +79,7 @@ export interface LocalMetabaseFanoutProgress {
|
|||
metabaseDatabaseId: number;
|
||||
targetConnectionId: string;
|
||||
jobId: string;
|
||||
status: 'done' | 'partial' | 'failed';
|
||||
status: 'done' | 'failed';
|
||||
}): void;
|
||||
}
|
||||
|
||||
|
|
@ -125,11 +122,10 @@ function findAdapter(adapters: SourceAdapter[], source: string): SourceAdapter {
|
|||
return adapter;
|
||||
}
|
||||
|
||||
function localJobContext(jobId: string, memoryFlow?: MemoryFlowEventSink, abortSignal?: AbortSignal): IngestJobContext {
|
||||
function localJobContext(jobId: string, memoryFlow?: MemoryFlowEventSink): IngestJobContext {
|
||||
return {
|
||||
jobId,
|
||||
...(memoryFlow ? { memoryFlow } : {}),
|
||||
...(abortSignal ? { abortSignal } : {}),
|
||||
startPhase() {
|
||||
return new LocalIngestPhase();
|
||||
},
|
||||
|
|
@ -161,7 +157,6 @@ async function runScheduledPullJob(options: {
|
|||
queryExecutor?: KtxSqlQueryExecutorPort;
|
||||
logger?: KtxLogger;
|
||||
embeddingProvider?: import('../../llm/types.js').KtxEmbeddingProvider | null;
|
||||
abortSignal?: AbortSignal;
|
||||
}): Promise<LocalIngestResult> {
|
||||
const runtime = createLocalBundleIngestRuntime(options);
|
||||
const jobId = options.jobId ?? runtime.nextJobId();
|
||||
|
|
@ -173,7 +168,7 @@ async function runScheduledPullJob(options: {
|
|||
trigger: options.trigger ?? 'manual_resync',
|
||||
bundleRef: { kind: 'scheduled_pull', config: options.pullConfig },
|
||||
},
|
||||
localJobContext(jobId, options.memoryFlow, options.abortSignal),
|
||||
localJobContext(jobId, options.memoryFlow),
|
||||
);
|
||||
const report = await runtime.store.findByJobId(jobId);
|
||||
if (!report) {
|
||||
|
|
@ -216,7 +211,6 @@ export async function runLocalIngest(options: RunLocalIngestOptions): Promise<Lo
|
|||
queryExecutor: options.queryExecutor,
|
||||
logger: options.logger,
|
||||
embeddingProvider: options.embeddingProvider,
|
||||
abortSignal: options.abortSignal,
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -228,7 +222,7 @@ export async function runLocalIngest(options: RunLocalIngestOptions): Promise<Lo
|
|||
trigger: options.trigger ?? (options.sourceDir ? 'upload' : 'manual_resync'),
|
||||
bundleRef,
|
||||
},
|
||||
localJobContext(jobId, options.memoryFlow, options.abortSignal),
|
||||
localJobContext(jobId, options.memoryFlow),
|
||||
);
|
||||
const report = await runtime.store.findByJobId(jobId);
|
||||
if (!report) {
|
||||
|
|
@ -238,11 +232,11 @@ export async function runLocalIngest(options: RunLocalIngestOptions): Promise<Lo
|
|||
}
|
||||
|
||||
function metabaseFanoutStatus(children: LocalMetabaseFanoutChild[]): LocalMetabaseFanoutResult['status'] {
|
||||
const outcomes = children.map((child) => ingestReportOutcome(child.report));
|
||||
if (outcomes.every((outcome) => outcome === 'done')) {
|
||||
const succeeded = children.filter((child) => child.report.body.failedWorkUnits.length === 0).length;
|
||||
if (succeeded === children.length) {
|
||||
return 'all_succeeded';
|
||||
}
|
||||
if (outcomes.every((outcome) => outcome === 'error')) {
|
||||
if (succeeded === 0) {
|
||||
return 'all_failed';
|
||||
}
|
||||
return 'partial_failure';
|
||||
|
|
@ -367,9 +361,6 @@ export async function runLocalMetabaseIngest(
|
|||
|
||||
const children: LocalMetabaseFanoutChild[] = [];
|
||||
for (const childPlan of childPlans) {
|
||||
if (options.abortSignal?.aborted) {
|
||||
throw createAbortError();
|
||||
}
|
||||
const targetConnectionId = safeSegment('target connection id', childPlan.targetConnectionId);
|
||||
if (!options.project.config.connections[targetConnectionId]) {
|
||||
throw new Error(`Target connection "${targetConnectionId}" is not configured in ktx.yaml`);
|
||||
|
|
@ -399,12 +390,8 @@ export async function runLocalMetabaseIngest(
|
|||
queryExecutor: options.queryExecutor,
|
||||
logger: options.logger,
|
||||
embeddingProvider: options.embeddingProvider,
|
||||
abortSignal: options.abortSignal,
|
||||
});
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) {
|
||||
throw error;
|
||||
}
|
||||
child = await recordLocalMetabaseChildFailure({
|
||||
project: options.project,
|
||||
jobId: childJobId,
|
||||
|
|
@ -414,13 +401,12 @@ export async function runLocalMetabaseIngest(
|
|||
error,
|
||||
});
|
||||
}
|
||||
const childOutcome = ingestReportOutcome(child.report);
|
||||
options.progress?.onMetabaseChildCompleted?.({
|
||||
metabaseConnectionId,
|
||||
metabaseDatabaseId: childPlan.metabaseDatabaseId,
|
||||
targetConnectionId,
|
||||
jobId: child.report.jobId,
|
||||
status: childOutcome === 'error' ? 'failed' : childOutcome,
|
||||
status: child.report.body.failedWorkUnits.length > 0 ? 'failed' : 'done',
|
||||
});
|
||||
children.push({
|
||||
jobId: child.report.jobId,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
import type { MemoryAction } from '../../../context/memory/types.js';
|
||||
import type { LocalIngestRunRecord } from '../local-stage-ingest.js';
|
||||
import { ingestReportOutcome } from '../reports.js';
|
||||
import type { IngestReportSnapshot } from '../reports.js';
|
||||
import type {
|
||||
MemoryFlowActionDetail,
|
||||
|
|
@ -73,7 +72,7 @@ function fullModeMetadata(input: {
|
|||
}
|
||||
|
||||
function reportStatus(report: IngestReportSnapshot): MemoryFlowReplayInput['status'] {
|
||||
return ingestReportOutcome(report) === 'error' ? 'error' : 'done';
|
||||
return report.body.failedWorkUnits.length > 0 ? 'error' : 'done';
|
||||
}
|
||||
|
||||
function reportCreatedEvent(report: IngestReportSnapshot): MemoryFlowEvent {
|
||||
|
|
|
|||
|
|
@ -70,13 +70,6 @@ const memoryFlowEventSchema = z.discriminatedUnion('type', [
|
|||
message: z.string().min(1),
|
||||
transient: z.boolean().optional(),
|
||||
}),
|
||||
eventSchema({
|
||||
type: z.literal('rate_limit_wait'),
|
||||
provider: z.string(),
|
||||
rateLimitType: z.string().optional(),
|
||||
resumeAtMs: z.number().int().nonnegative(),
|
||||
remainingMs: z.number().int().nonnegative(),
|
||||
}),
|
||||
eventSchema({
|
||||
type: z.literal('work_unit_started'),
|
||||
unitKey: z.string().min(1),
|
||||
|
|
|
|||
|
|
@ -60,13 +60,6 @@ type MemoryFlowEventPayload =
|
|||
message: string;
|
||||
transient?: boolean;
|
||||
}
|
||||
| {
|
||||
type: 'rate_limit_wait';
|
||||
provider: string;
|
||||
rateLimitType?: string;
|
||||
resumeAtMs: number;
|
||||
remainingMs: number;
|
||||
}
|
||||
| {
|
||||
type: 'work_unit_started';
|
||||
unitKey: string;
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ import type { KtxFileStorePort } from '../../context/core/file-store.js';
|
|||
import type { KtxLogger } from '../../context/core/config.js';
|
||||
import type { SessionOutcome } from '../../context/core/session-worktree.service.js';
|
||||
import type { AgentRunnerPort, KtxLlmRuntimePort, KtxRuntimeToolSet } from '../../context/llm/runtime-port.js';
|
||||
import type { RateLimitGovernor } from '../llm/rate-limit-governor.js';
|
||||
import type { MemoryAction, MemoryKnowledgeSlRefsPort } from '../../context/memory/types.js';
|
||||
import type { PromptService } from '../../context/prompts/prompt.service.js';
|
||||
import type { SkillsRegistryService } from '../../context/skills/skills-registry.service.js';
|
||||
|
|
@ -145,9 +144,6 @@ interface IngestSettingsPort {
|
|||
workUnitMaxConcurrency?: number;
|
||||
workUnitStepBudget?: number;
|
||||
workUnitFailureMode?: 'abort' | 'continue';
|
||||
rateLimitGovernor?: RateLimitGovernor;
|
||||
/** Print a timing breakdown to stderr at the end of each run (config-driven; see also KTX_PROFILE_INGEST). `'json'` emits the raw structured profile. */
|
||||
profileIngest?: boolean | 'json';
|
||||
ingestTraceLevel?: IngestTraceLevel;
|
||||
}
|
||||
|
||||
|
|
@ -325,7 +321,6 @@ export interface CuratorPaginationPort {
|
|||
buildToolSet: (passNumber: number) => KtxRuntimeToolSet;
|
||||
getReconciliationActions: () => MemoryAction[];
|
||||
onStepFinish?: (info: { passNumber: number; stepIndex: number; stepBudget: number }) => void;
|
||||
abortSignal?: AbortSignal;
|
||||
}): Promise<ReconciliationOutcome & { report: CuratorPaginationReport; warnings: string[] }>;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -146,20 +146,6 @@ export function savedMemoryCountsForReport(report: IngestReportSnapshot): Ingest
|
|||
};
|
||||
}
|
||||
|
||||
/** @internal */
|
||||
export type IngestReportOutcome = 'done' | 'partial' | 'error';
|
||||
|
||||
export function ingestReportOutcome(report: IngestReportSnapshot): IngestReportOutcome {
|
||||
if (report.body.status === 'failed') {
|
||||
return 'error';
|
||||
}
|
||||
if (report.body.failedWorkUnits.length === 0) {
|
||||
return 'done';
|
||||
}
|
||||
const { wikiCount, slCount } = savedMemoryCountsForReport(report);
|
||||
return wikiCount + slCount > 0 ? 'partial' : 'error';
|
||||
}
|
||||
|
||||
export function buildStageIndexFromReportBody(jobId: string, connectionId: string, body: IngestReportBody): StageIndex {
|
||||
return {
|
||||
jobId,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
import type { KtxModelRole } from '../../../llm/types.js';
|
||||
import { isAbortError } from '../../core/abort.js';
|
||||
import type { AgentRunnerPort, KtxRuntimeToolSet, RunLoopMetrics } from '../../../context/llm/runtime-port.js';
|
||||
import type { AgentRunnerPort, KtxRuntimeToolSet } from '../../../context/llm/runtime-port.js';
|
||||
import type { CaptureSession, MemoryAction } from '../../../context/memory/types.js';
|
||||
import { listTouchedSlSources, type TouchedSlSource } from '../../../context/tools/touched-sl-sources.js';
|
||||
import type { WorkUnit } from '../types.js';
|
||||
|
|
@ -29,7 +28,6 @@ export interface WorkUnitExecutionDeps {
|
|||
connectionId: string;
|
||||
jobId: string;
|
||||
onStepFinish?: (info: { stepIndex: number; stepBudget: number }) => void;
|
||||
abortSignal?: AbortSignal;
|
||||
toolFailureCount?: (unitKey: string) => number;
|
||||
}
|
||||
|
||||
|
|
@ -46,8 +44,6 @@ export interface WorkUnitOutcome {
|
|||
patchPath?: string;
|
||||
patchTouchedPaths?: string[];
|
||||
childWorktreePath?: string;
|
||||
/** Timing and token metrics for the work-unit agent loop, used for ingest profiling. */
|
||||
metrics?: RunLoopMetrics;
|
||||
}
|
||||
|
||||
export async function executeWorkUnit(deps: WorkUnitExecutionDeps, wu: WorkUnit): Promise<WorkUnitOutcome> {
|
||||
|
|
@ -108,12 +104,8 @@ export async function executeWorkUnit(deps: WorkUnitExecutionDeps, wu: WorkUnit)
|
|||
jobId: deps.jobId,
|
||||
},
|
||||
onStepFinish: deps.onStepFinish,
|
||||
abortSignal: deps.abortSignal,
|
||||
});
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) {
|
||||
throw error;
|
||||
}
|
||||
return failWithResetFromCurrentHead(error instanceof Error ? error.message : String(error));
|
||||
}
|
||||
|
||||
|
|
@ -133,7 +125,6 @@ export async function executeWorkUnit(deps: WorkUnitExecutionDeps, wu: WorkUnit)
|
|||
touchedSlSources: [],
|
||||
slDisallowed: wu.slDisallowed,
|
||||
slDisallowedReason: wu.slDisallowedReason,
|
||||
...(runResult.metrics ? { metrics: runResult.metrics } : {}),
|
||||
};
|
||||
};
|
||||
|
||||
|
|
@ -171,6 +162,5 @@ export async function executeWorkUnit(deps: WorkUnitExecutionDeps, wu: WorkUnit)
|
|||
touchedSlSources: touched,
|
||||
slDisallowed: wu.slDisallowed,
|
||||
slDisallowedReason: wu.slDisallowedReason,
|
||||
...(runResult.metrics ? { metrics: runResult.metrics } : {}),
|
||||
};
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import type { AgentRunnerPort, KtxRuntimeToolSet, RunLoopMetrics } from '../../../context/llm/runtime-port.js';
|
||||
import type { AgentRunnerPort, KtxRuntimeToolSet } from '../../../context/llm/runtime-port.js';
|
||||
import type { KtxModelRole } from '../../../llm/types.js';
|
||||
import type { EvictionUnit } from '../types.js';
|
||||
import type { StageIndex } from './stage-index.types.js';
|
||||
|
|
@ -16,7 +16,6 @@ export interface ReconciliationContext {
|
|||
jobId: string;
|
||||
force?: boolean;
|
||||
onStepFinish?: (info: { stepIndex: number; stepBudget: number }) => void;
|
||||
abortSignal?: AbortSignal;
|
||||
forceRun?: boolean;
|
||||
}
|
||||
|
||||
|
|
@ -24,7 +23,6 @@ export interface ReconciliationOutcome {
|
|||
skipped: boolean;
|
||||
stopReason?: 'budget' | 'natural' | 'error';
|
||||
error?: Error;
|
||||
metrics?: RunLoopMetrics;
|
||||
}
|
||||
|
||||
export async function runReconciliationStage4(ctx: ReconciliationContext): Promise<ReconciliationOutcome> {
|
||||
|
|
@ -41,7 +39,6 @@ export async function runReconciliationStage4(ctx: ReconciliationContext): Promi
|
|||
stepBudget: ctx.stepBudget,
|
||||
telemetryTags: { operationName: 'ingest-bundle-reconcile', source: ctx.sourceKey, jobId: ctx.jobId },
|
||||
onStepFinish: ctx.onStepFinish,
|
||||
abortSignal: ctx.abortSignal,
|
||||
});
|
||||
return { skipped: false, stopReason: run.stopReason, error: run.error, ...(run.metrics ? { metrics: run.metrics } : {}) };
|
||||
return { skipped: false, stopReason: run.stopReason, error: run.error };
|
||||
}
|
||||
|
|
|
|||
|
|
@ -81,13 +81,8 @@ export function wrapToolsWithLogger<T extends KtxRuntimeToolSet>(
|
|||
return wrapped as T;
|
||||
}
|
||||
|
||||
// Fire-and-forget appends are intentional (the agent hot path must never block
|
||||
// or fail on logging), but readers like the ingest profiler need to know when
|
||||
// the writes have settled. Track in-flight appends so a consumer can flush.
|
||||
const pendingWrites = new Set<Promise<void>>();
|
||||
|
||||
function appendEntry(path: string, entry: ToolCallLogEntry): void {
|
||||
const write = (async () => {
|
||||
void (async () => {
|
||||
try {
|
||||
await mkdir(dirname(path), { recursive: true });
|
||||
await appendFile(path, `${safeStringify(entry)}\n`, 'utf-8');
|
||||
|
|
@ -95,37 +90,6 @@ function appendEntry(path: string, entry: ToolCallLogEntry): void {
|
|||
// best-effort
|
||||
}
|
||||
})();
|
||||
pendingWrites.add(write);
|
||||
void write.finally(() => pendingWrites.delete(write));
|
||||
}
|
||||
|
||||
/**
|
||||
* Await all in-flight tool-call log writes (best-effort, bounded by `timeoutMs`
|
||||
* so it can never hang a caller). Lets readers such as the ingest profiler see
|
||||
* complete transcripts despite the fire-and-forget append design.
|
||||
*/
|
||||
export async function flushToolCallLogs(timeoutMs = 5000): Promise<void> {
|
||||
const pending = [...pendingWrites];
|
||||
if (pending.length === 0) {
|
||||
return;
|
||||
}
|
||||
const settled = Promise.allSettled(pending).then(() => undefined);
|
||||
if (timeoutMs <= 0) {
|
||||
await settled;
|
||||
return;
|
||||
}
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
const timeout = new Promise<void>((resolve) => {
|
||||
timer = setTimeout(resolve, timeoutMs);
|
||||
timer.unref?.();
|
||||
});
|
||||
try {
|
||||
await Promise.race([settled, timeout]);
|
||||
} finally {
|
||||
if (timer) {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function safeStringify(v: unknown): string {
|
||||
|
|
|
|||
|
|
@ -220,6 +220,5 @@ export interface IngestJobPhase {
|
|||
export interface IngestJobContext {
|
||||
jobId: string;
|
||||
memoryFlow?: MemoryFlowEventSink;
|
||||
abortSignal?: AbortSignal;
|
||||
startPhase(weight: number): IngestJobPhase;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,15 +3,12 @@ import type { KtxLlmProvider } from '../../llm/types.js';
|
|||
import { generateText, Output, stepCountIs, type FlexibleSchema, type TelemetrySettings, type ToolSet } from 'ai';
|
||||
import type { z } from 'zod';
|
||||
import { noopLogger, type KtxLogger } from '../../context/core/config.js';
|
||||
import { isAbortError } from '../core/abort.js';
|
||||
import { summarizeKtxLlmDebugRequest, type KtxLlmDebugRequestRecorder } from './debug-request-recorder.js';
|
||||
import type { RateLimitGovernor, RateLimitProvider, RateLimitSignal } from './rate-limit-governor.js';
|
||||
import { createAiSdkToolSet } from './runtime-tools.js';
|
||||
import type {
|
||||
KtxGenerateObjectInput,
|
||||
KtxGenerateTextInput,
|
||||
KtxLlmRuntimePort,
|
||||
LlmTokenUsage,
|
||||
RunLoopParams,
|
||||
RunLoopResult,
|
||||
} from './runtime-port.js';
|
||||
|
|
@ -20,151 +17,17 @@ interface AgentTelemetryPort {
|
|||
createTelemetry(tags: Record<string, string>): TelemetrySettings;
|
||||
}
|
||||
|
||||
interface MaybeUsage {
|
||||
inputTokens?: number;
|
||||
outputTokens?: number;
|
||||
totalTokens?: number;
|
||||
}
|
||||
|
||||
function toLlmTokenUsage(usage: MaybeUsage | undefined): LlmTokenUsage {
|
||||
if (!usage) {
|
||||
return {};
|
||||
}
|
||||
return {
|
||||
...(usage.inputTokens !== undefined ? { inputTokens: usage.inputTokens } : {}),
|
||||
...(usage.outputTokens !== undefined ? { outputTokens: usage.outputTokens } : {}),
|
||||
...(usage.totalTokens !== undefined ? { totalTokens: usage.totalTokens } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export interface AiSdkKtxLlmRuntimeDeps {
|
||||
llmProvider: KtxLlmProvider;
|
||||
telemetry?: AgentTelemetryPort;
|
||||
logger?: KtxLogger;
|
||||
debugRequestRecorder?: KtxLlmDebugRequestRecorder;
|
||||
rateLimitGovernor?: Pick<RateLimitGovernor, 'waitForReady' | 'report' | 'maxRetryAttempts'>;
|
||||
}
|
||||
|
||||
function hasTools(tools: Record<string, unknown>): boolean {
|
||||
return Object.keys(tools).length > 0;
|
||||
}
|
||||
|
||||
function modelProviderName(model: unknown): RateLimitProvider {
|
||||
const provider = (model as { provider?: string }).provider ?? '';
|
||||
return provider.includes('vertex') || provider.includes('google') ? 'vertex' : 'anthropic-api';
|
||||
}
|
||||
|
||||
interface HeaderLimitPair {
|
||||
limit: string;
|
||||
remaining: string;
|
||||
rateLimitType: string;
|
||||
}
|
||||
|
||||
const RATE_LIMIT_HEADER_PAIRS: HeaderLimitPair[] = [
|
||||
{
|
||||
limit: 'anthropic-ratelimit-requests-limit',
|
||||
remaining: 'anthropic-ratelimit-requests-remaining',
|
||||
rateLimitType: 'rpm',
|
||||
},
|
||||
{
|
||||
limit: 'anthropic-ratelimit-tokens-limit',
|
||||
remaining: 'anthropic-ratelimit-tokens-remaining',
|
||||
rateLimitType: 'tpm',
|
||||
},
|
||||
{
|
||||
limit: 'anthropic-ratelimit-input-tokens-limit',
|
||||
remaining: 'anthropic-ratelimit-input-tokens-remaining',
|
||||
rateLimitType: 'itpm',
|
||||
},
|
||||
{
|
||||
limit: 'anthropic-ratelimit-output-tokens-limit',
|
||||
remaining: 'anthropic-ratelimit-output-tokens-remaining',
|
||||
rateLimitType: 'otpm',
|
||||
},
|
||||
{
|
||||
limit: 'x-ratelimit-limit-requests',
|
||||
remaining: 'x-ratelimit-remaining-requests',
|
||||
rateLimitType: 'rpm',
|
||||
},
|
||||
{
|
||||
limit: 'x-ratelimit-limit-tokens',
|
||||
remaining: 'x-ratelimit-remaining-tokens',
|
||||
rateLimitType: 'tpm',
|
||||
},
|
||||
];
|
||||
|
||||
function normalizeHeaders(headers: unknown): Record<string, string> {
|
||||
if (!headers || typeof headers !== 'object') {
|
||||
return {};
|
||||
}
|
||||
const get = (headers as { get?: unknown }).get;
|
||||
if (typeof get === 'function') {
|
||||
const out: Record<string, string> = {};
|
||||
for (const pair of RATE_LIMIT_HEADER_PAIRS) {
|
||||
const limit = get.call(headers, pair.limit);
|
||||
const remaining = get.call(headers, pair.remaining);
|
||||
if (typeof limit === 'string') out[pair.limit] = limit;
|
||||
if (typeof remaining === 'string') out[pair.remaining] = remaining;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
return Object.fromEntries(
|
||||
Object.entries(headers as Record<string, unknown>)
|
||||
.filter((entry): entry is [string, string | number] => typeof entry[1] === 'string' || typeof entry[1] === 'number')
|
||||
.map(([key, value]) => [key.toLowerCase(), String(value)]),
|
||||
);
|
||||
}
|
||||
|
||||
function numericHeader(headers: Record<string, string>, key: string): number | undefined {
|
||||
const value = Number(headers[key]);
|
||||
return Number.isFinite(value) && value >= 0 ? value : undefined;
|
||||
}
|
||||
|
||||
function utilizationForPair(headers: Record<string, string>, pair: HeaderLimitPair): number | undefined {
|
||||
const limit = numericHeader(headers, pair.limit);
|
||||
const remaining = numericHeader(headers, pair.remaining);
|
||||
if (limit === undefined || remaining === undefined || limit <= 0) {
|
||||
return undefined;
|
||||
}
|
||||
return 1 - Math.min(limit, remaining) / limit;
|
||||
}
|
||||
|
||||
function aiSdkHeaderRateLimitSignal(provider: RateLimitProvider, result: unknown): RateLimitSignal | undefined {
|
||||
const headers = normalizeHeaders((result as { response?: { headers?: unknown } }).response?.headers);
|
||||
let best: { utilization: number; rateLimitType: string } | undefined;
|
||||
for (const pair of RATE_LIMIT_HEADER_PAIRS) {
|
||||
const utilization = utilizationForPair(headers, pair);
|
||||
if (utilization === undefined) {
|
||||
continue;
|
||||
}
|
||||
if (!best || utilization > best.utilization) {
|
||||
best = { utilization, rateLimitType: pair.rateLimitType };
|
||||
}
|
||||
}
|
||||
if (!best) {
|
||||
return undefined;
|
||||
}
|
||||
return {
|
||||
provider,
|
||||
status: 'allowed',
|
||||
rateLimitType: best.rateLimitType,
|
||||
utilization: Number(best.utilization.toFixed(4)),
|
||||
};
|
||||
}
|
||||
|
||||
function retryAfterMs(error: unknown): number | undefined {
|
||||
const value = (error as { retryAfter?: unknown }).retryAfter;
|
||||
if (typeof value === 'number' && Number.isFinite(value) && value > 0) {
|
||||
return value < 1_000 ? value * 1_000 : value;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function isAiSdkRateLimitError(error: unknown): boolean {
|
||||
const record = error as { name?: string; statusCode?: number; status?: number };
|
||||
return record.name === 'TooManyRequestsError' || record.statusCode === 429 || record.status === 429;
|
||||
}
|
||||
|
||||
export class AiSdkKtxLlmRuntime implements KtxLlmRuntimePort {
|
||||
private readonly logger: KtxLogger;
|
||||
|
||||
|
|
@ -172,41 +35,6 @@ export class AiSdkKtxLlmRuntime implements KtxLlmRuntimePort {
|
|||
this.logger = deps.logger ?? noopLogger;
|
||||
}
|
||||
|
||||
private async generateTextWithRateLimitRetry<T>(
|
||||
provider: RateLimitProvider,
|
||||
abortSignal: AbortSignal | undefined,
|
||||
run: () => Promise<T>,
|
||||
): Promise<T> {
|
||||
// maxRetryAttempts() returns 1 when no governor is present or pacing is
|
||||
// disabled, so a 429 throws immediately instead of hammering the provider
|
||||
// with no backoff; the AI SDK's own maxRetries still handles transient 429s.
|
||||
const maxAttempts = this.deps.rateLimitGovernor?.maxRetryAttempts() ?? 1;
|
||||
let attempt = 0;
|
||||
while (true) {
|
||||
await this.deps.rateLimitGovernor?.waitForReady(abortSignal);
|
||||
try {
|
||||
const result = await run();
|
||||
const signal = aiSdkHeaderRateLimitSignal(provider, result);
|
||||
if (signal) {
|
||||
this.deps.rateLimitGovernor?.report(signal);
|
||||
}
|
||||
return result;
|
||||
} catch (error) {
|
||||
if (isAbortError(error) || !isAiSdkRateLimitError(error) || attempt >= maxAttempts - 1) {
|
||||
throw error;
|
||||
}
|
||||
attempt += 1;
|
||||
const retryAfter = retryAfterMs(error);
|
||||
this.deps.rateLimitGovernor?.report({
|
||||
provider,
|
||||
status: 'rejected',
|
||||
rateLimitType: 'http_429',
|
||||
...(retryAfter !== undefined ? { retryAfterMs: retryAfter } : {}),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async generateText(input: KtxGenerateTextInput): Promise<string> {
|
||||
const model = this.deps.llmProvider.getModel(input.role);
|
||||
if ((model as { provider?: string }).provider === 'deterministic') {
|
||||
|
|
@ -220,14 +48,12 @@ export class AiSdkKtxLlmRuntime implements KtxLlmRuntimePort {
|
|||
model,
|
||||
});
|
||||
const split = splitKtxSystemMessages(built.messages);
|
||||
const startedAt = Date.now();
|
||||
const request = {
|
||||
const result = await generateText({
|
||||
model,
|
||||
temperature: input.temperature ?? 0,
|
||||
...(split.system ? { system: split.system } : {}),
|
||||
messages: split.messages,
|
||||
tools: built.tools as ToolSet,
|
||||
...(input.abortSignal ? { abortSignal: input.abortSignal } : {}),
|
||||
...(hasTools(tools)
|
||||
? {
|
||||
experimental_repairToolCall: this.deps.llmProvider.repairToolCallHandler({
|
||||
|
|
@ -235,9 +61,7 @@ export class AiSdkKtxLlmRuntime implements KtxLlmRuntimePort {
|
|||
}),
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
const result = await this.generateTextWithRateLimitRetry(modelProviderName(model), input.abortSignal, () => generateText(request));
|
||||
input.onMetrics?.({ totalMs: Date.now() - startedAt, usage: toLlmTokenUsage(result.totalUsage ?? result.usage) });
|
||||
});
|
||||
if (typeof result.text !== 'string') {
|
||||
throw new Error('KTX LLM text generation returned no text');
|
||||
}
|
||||
|
|
@ -256,14 +80,12 @@ export class AiSdkKtxLlmRuntime implements KtxLlmRuntimePort {
|
|||
model,
|
||||
});
|
||||
const split = splitKtxSystemMessages(built.messages);
|
||||
const startedAt = Date.now();
|
||||
const request = {
|
||||
const result = await generateText({
|
||||
model,
|
||||
temperature: input.temperature ?? 0,
|
||||
...(split.system ? { system: split.system } : {}),
|
||||
messages: split.messages,
|
||||
tools: built.tools as ToolSet,
|
||||
...(input.abortSignal ? { abortSignal: input.abortSignal } : {}),
|
||||
...(hasTools(tools)
|
||||
? {
|
||||
experimental_repairToolCall: this.deps.llmProvider.repairToolCallHandler({
|
||||
|
|
@ -272,9 +94,7 @@ export class AiSdkKtxLlmRuntime implements KtxLlmRuntimePort {
|
|||
}
|
||||
: {}),
|
||||
output: Output.object({ schema: input.schema as unknown as FlexibleSchema<TOutput> }),
|
||||
};
|
||||
const result = await this.generateTextWithRateLimitRetry(modelProviderName(model), input.abortSignal, () => generateText(request));
|
||||
input.onMetrics?.({ totalMs: Date.now() - startedAt, usage: toLlmTokenUsage(result.totalUsage ?? result.usage) });
|
||||
});
|
||||
if (result.output == null) {
|
||||
throw new Error('KTX LLM object generation returned no output');
|
||||
}
|
||||
|
|
@ -283,8 +103,6 @@ export class AiSdkKtxLlmRuntime implements KtxLlmRuntimePort {
|
|||
|
||||
async runAgentLoop(params: RunLoopParams): Promise<RunLoopResult> {
|
||||
let stepIndex = 0;
|
||||
const startedAt = Date.now();
|
||||
const stepBoundariesMs: number[] = [];
|
||||
try {
|
||||
const model = this.deps.llmProvider.getModel(params.modelRole);
|
||||
const tools = createAiSdkToolSet(params.toolSet);
|
||||
|
|
@ -310,7 +128,7 @@ export class AiSdkKtxLlmRuntime implements KtxLlmRuntimePort {
|
|||
}),
|
||||
);
|
||||
|
||||
const request = {
|
||||
await generateText({
|
||||
model,
|
||||
temperature: 0,
|
||||
stopWhen: stepCountIs(params.stepBudget),
|
||||
|
|
@ -321,10 +139,8 @@ export class AiSdkKtxLlmRuntime implements KtxLlmRuntimePort {
|
|||
...(promptMessages.system ? { system: promptMessages.system } : {}),
|
||||
messages: promptMessages.messages,
|
||||
tools: built.tools as ToolSet,
|
||||
...(params.abortSignal ? { abortSignal: params.abortSignal } : {}),
|
||||
onStepFinish: async () => {
|
||||
stepIndex += 1;
|
||||
stepBoundariesMs.push(Date.now() - startedAt);
|
||||
if (!params.onStepFinish) {
|
||||
return;
|
||||
}
|
||||
|
|
@ -338,28 +154,12 @@ export class AiSdkKtxLlmRuntime implements KtxLlmRuntimePort {
|
|||
);
|
||||
}
|
||||
},
|
||||
};
|
||||
const result = await this.generateTextWithRateLimitRetry(modelProviderName(model), params.abortSignal, () => generateText(request));
|
||||
return {
|
||||
stopReason: 'natural',
|
||||
metrics: {
|
||||
totalMs: Date.now() - startedAt,
|
||||
stepCount: stepIndex,
|
||||
stepBoundariesMs,
|
||||
usage: toLlmTokenUsage(result.totalUsage ?? result.usage),
|
||||
},
|
||||
};
|
||||
});
|
||||
return { stopReason: 'natural' };
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) {
|
||||
throw error;
|
||||
}
|
||||
const err = error instanceof Error ? error : new Error(String(error));
|
||||
this.logger.warn(`[agent-runner] loop failed: ${err.message}`);
|
||||
return {
|
||||
stopReason: 'error',
|
||||
error: err,
|
||||
metrics: { totalMs: Date.now() - startedAt, stepCount: stepIndex, stepBoundariesMs, usage: {} },
|
||||
};
|
||||
return { stopReason: 'error', error: err };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue